From f4b349326d438f282862572948b48a3aa92f32a6 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Fri, 31 Jul 2026 19:24:43 -0400 Subject: [PATCH 01/96] feat: add unrealpak footer+index reader (#136) --- internal/unrealpak/pak.go | 166 +++++++++++++++ internal/unrealpak/reader.go | 339 ++++++++++++++++++++++++++++++ internal/unrealpak/reader_test.go | 189 +++++++++++++++++ 3 files changed, 694 insertions(+) create mode 100644 internal/unrealpak/pak.go create mode 100644 internal/unrealpak/reader.go create mode 100644 internal/unrealpak/reader_test.go diff --git a/internal/unrealpak/pak.go b/internal/unrealpak/pak.go new file mode 100644 index 0000000..4c4b76d --- /dev/null +++ b/internal/unrealpak/pak.go @@ -0,0 +1,166 @@ +package unrealpak + +import ( + "bytes" + "crypto/sha1" //nolint:gosec // pak format uses SHA1, not our choice + "encoding/binary" + "errors" + "strings" + "unicode/utf16" +) + +// ErrUnsupportedFormat indicates the pak uses a feature this package +// deliberately does not support (compression, encryption, exotic FString +// encodings) rather than a genuine parse failure. Callers should fail loudly +// on this, not silently degrade (repo precedent: #95). +var ErrUnsupportedFormat = errors.New("unrealpak: unsupported pak feature") + +const magic uint32 = 0x5A6F12E1 + +// footerSize is the only footer shape this package supports: the version>=8 +// layout, EncryptionKeyGuid(16)+bEncryptedIndex(1)+Magic(4)+Version(4)+ +// IndexOffset(8)+IndexSize(8)+IndexHash(20)+CompressionMethods(5x32) = 221. +// Note EncryptionKeyGuid and bEncryptedIndex precede Magic — not after +// IndexHash, as some public docs describe. Confirmed on all 34 paks in a real +// Icarus install (Task 1); see docs/plans/icarus-pak-format-findings.md. +const footerSize = 221 + +// minVersion is the oldest pak version this package reads. Version 10 +// (PakFile_Version_PathHashIndex) introduced the three-part index — primary +// index + path-hash index + full directory index — that this package parses. +// Older paks use a flat index with a completely different shape; rather than +// carry a second parser for a layout Icarus does not ship, they are a hard +// ErrUnsupportedFormat (repo precedent #95: no silent fallbacks). +const minVersion int32 = 10 + +// writeVersion is what Writer emits: the same version Icarus's own paks use, +// so the engine loads our output through the exact code path it already uses. +const writeVersion int32 = 11 + +// storedHeaderSize is the on-disk size of the per-entry FPakEntry header that +// precedes each stored (uncompressed) file's payload: +// Offset(8)+Size(8)+UncompressedSize(8)+CompressionMethodIndex(4)+Hash(20)+ +// Flags(1)+CompressionBlockSize(4) = 53. Compressed entries add +// BlockCount(4)+16*blocks between Hash and Flags; this package never writes +// those and refuses to read their payloads. +const storedHeaderSize = 53 + +// FileEntry describes one file inside a pak, as returned by Reader.Files. +type FileEntry struct { + Path string // Mount-relative path, e.g. "Icarus/Content/Data/AI-D_AIGrowth.json" + Size int64 // Uncompressed size in bytes +} + +// hashPath computes a path's key in the pak's path-hash index: FNV-1a 64 over +// the UTF-16LE bytes of the lowercased mount-relative path (no NUL +// terminator), seeded by ADDING the pak's PathHashSeed to the FNV offset +// basis. Any leading "/" is stripped first — the full directory index stores +// root-level files under a "/" directory, and the hash is taken over the path +// without it. +// +// This recipe was not guessed: it was recovered by brute-forcing seed/ +// encoding/case/prefix combinations until computed hashes matched stored keys, +// then verified against all 173,078 entries across all 34 paks in a real +// install. See docs/plans/icarus-pak-format-findings.md. +// +// strings.ToLower is full-Unicode where UE's FChar::ToLower is not, but no +// non-ASCII path exists in any shipped Icarus pak and this package controls +// the paths it writes, so the two agree for everything we handle. +func hashPath(mountRelative string, seed uint64) uint64 { + const ( + offsetBasis uint64 = 0xCBF29CE484222325 + prime uint64 = 0x00000100000001B3 + ) + h := offsetBasis + seed + for _, u := range utf16.Encode([]rune(strings.ToLower(strings.TrimPrefix(mountRelative, "/")))) { + h ^= uint64(byte(u)) + h *= prime + h ^= uint64(byte(u >> 8)) + h *= prime + } + return h +} + +// defaultMountPoint is the mount point Writer stamps into the primary index. +// Icarus's own data.pak uses an absolute cook-machine path +// ("C:/BA/work/.../Temp/Data/"); "../../../" is the conventional relative form +// used by its pakchunks. Confirming which one a _P.pak needs to override +// Content/Data/data.pak in-game is a post-plan validation item. +const defaultMountPoint = "../../../" + +// writeFString writes a length-prefixed ANSI Unreal FString (length includes +// the trailing NUL). +func writeFString(buf *bytes.Buffer, s string) { + b := append([]byte(s), 0) + binary.Write(buf, binary.LittleEndian, int32(len(b))) //nolint:errcheck // bytes.Buffer writes never fail + buf.Write(b) +} + +// splitMountPath splits a mount-relative path into the directory-index key +// (trailing "/", or exactly "/" for a root-level file) and the leaf name, +// matching how real paks key their directory indexes. +func splitMountPath(rel string) (dir, file string) { + if i := strings.LastIndex(rel, "/"); i >= 0 { + return rel[:i+1], rel[i+1:] + } + return "/", rel +} + +// storedEntryHeader builds the 53-byte FPakEntry header that precedes a stored +// file's payload on disk. The Offset field is always 0 in this local copy — +// real paks write 0 there too, the authoritative offset lives in the index. +// Hash is the SHA1 of the on-disk payload bytes. +func storedEntryHeader(size int64, content []byte) []byte { + var b bytes.Buffer + binary.Write(&b, binary.LittleEndian, int64(0)) //nolint:errcheck // Offset + binary.Write(&b, binary.LittleEndian, size) //nolint:errcheck // Size + binary.Write(&b, binary.LittleEndian, size) //nolint:errcheck // UncompressedSize + binary.Write(&b, binary.LittleEndian, int32(0)) //nolint:errcheck // CompressionMethodIndex: stored + h := sha1.Sum(content) //nolint:gosec + b.Write(h[:]) + b.WriteByte(0) // Flags: not encrypted, not deleted + binary.Write(&b, binary.LittleEndian, uint32(0)) //nolint:errcheck // CompressionBlockSize + return b.Bytes() +} + +// buildPrimaryIndex serializes the primary index. Callers build it twice: the +// sub-index offsets it records point past its own end, but its length does not +// depend on their values (they are fixed-width int64), so a first pass with +// zero offsets measures it and a second pass writes the real ones. +func buildPrimaryIndex(numEntries int32, seed uint64, + phiOffset, phiSize int64, phiHash [20]byte, + fdiOffset, fdiSize int64, fdiHash [20]byte, encoded []byte) []byte { + var b bytes.Buffer + writeFString(&b, defaultMountPoint) + binary.Write(&b, binary.LittleEndian, numEntries) //nolint:errcheck + binary.Write(&b, binary.LittleEndian, seed) //nolint:errcheck // PathHashSeed + binary.Write(&b, binary.LittleEndian, int32(1)) //nolint:errcheck // bHasPathHashIndex + binary.Write(&b, binary.LittleEndian, phiOffset) //nolint:errcheck + binary.Write(&b, binary.LittleEndian, phiSize) //nolint:errcheck + b.Write(phiHash[:]) + binary.Write(&b, binary.LittleEndian, int32(1)) //nolint:errcheck // bHasFullDirectoryIndex + binary.Write(&b, binary.LittleEndian, fdiOffset) //nolint:errcheck + binary.Write(&b, binary.LittleEndian, fdiSize) //nolint:errcheck + b.Write(fdiHash[:]) + binary.Write(&b, binary.LittleEndian, int32(len(encoded))) //nolint:errcheck // EncodedPakEntriesSize + b.Write(encoded) + binary.Write(&b, binary.LittleEndian, int32(0)) //nolint:errcheck // NumNonEncodedFiles: none + return b.Bytes() +} + +// buildFooter serializes the 221-byte version>=8 footer. +func buildFooter(version int32, indexOffset, indexSize int64, indexHash [20]byte) []byte { + var b bytes.Buffer + b.Write(make([]byte, 16)) // EncryptionKeyGuid: zero + b.WriteByte(0) // bEncryptedIndex: false + binary.Write(&b, binary.LittleEndian, magic) //nolint:errcheck + binary.Write(&b, binary.LittleEndian, version) //nolint:errcheck + binary.Write(&b, binary.LittleEndian, indexOffset) //nolint:errcheck + binary.Write(&b, binary.LittleEndian, indexSize) //nolint:errcheck + b.Write(indexHash[:]) + // CompressionMethods: 5 fixed-width 32-byte name slots, all empty since + // this package only ever writes stored entries. Real paks name "Oodle" + // and "Zlib" here; an all-zero table is the correct shape for method 0. + b.Write(make([]byte, 160)) + return b.Bytes() +} diff --git a/internal/unrealpak/reader.go b/internal/unrealpak/reader.go new file mode 100644 index 0000000..2d1bddc --- /dev/null +++ b/internal/unrealpak/reader.go @@ -0,0 +1,339 @@ +package unrealpak + +import ( + "bytes" + "crypto/sha1" //nolint:gosec // pak format uses SHA1, not our choice + "encoding/binary" + "fmt" + "io" + "os" + "sort" + "strings" +) + +// Reader provides read access to an uncompressed, unencrypted UE4-range pak. +type Reader struct { + f *os.File + entries []readerEntry +} + +type readerEntry struct { + FileEntry + offset int64 // absolute offset of the entry's on-disk header + method int32 // CompressionMethodIndex; 0 = stored. Non-zero entries are + // enumerated but their payloads cannot be read (see ReadFile, Task 3). +} + +// Open parses path's footer and index. It does not read file contents — +// call ReadFile for that (Task 3). +func Open(path string) (*Reader, error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("unrealpak: opening %s: %w", path, err) + } + info, err := f.Stat() + if err != nil { + f.Close() //nolint:errcheck + return nil, fmt.Errorf("unrealpak: stat %s: %w", path, err) + } + + ft, err := readFooter(f, info.Size()) + if err != nil { + f.Close() //nolint:errcheck + return nil, err + } + if ft.encryptedIndex { + f.Close() //nolint:errcheck + return nil, fmt.Errorf("unrealpak: %s: %w: encrypted index", path, ErrUnsupportedFormat) + } + + indexBuf, err := readRegion(f, ft.indexOffset, ft.indexSize, ft.indexHash) + if err != nil { + f.Close() //nolint:errcheck + return nil, fmt.Errorf("unrealpak: %s: primary index: %w", path, err) + } + + entries, err := parseIndex(f, indexBuf) + if err != nil { + f.Close() //nolint:errcheck + return nil, fmt.Errorf("unrealpak: %s: parsing index: %w", path, err) + } + + return &Reader{f: f, entries: entries}, nil +} + +// readRegion reads size bytes at offset and verifies them against want. Every +// index region in a version-11 pak is SHA1-gated: the footer covers the +// primary index, and the primary index covers each sub-index. All three gates +// are enforced — a mismatch is corruption or an unrecognized layout, never +// something to parse through. +func readRegion(r io.ReaderAt, offset, size int64, want [20]byte) ([]byte, error) { + if offset < 0 || size < 0 { + return nil, fmt.Errorf("%w: negative region offset/size", ErrUnsupportedFormat) + } + buf := make([]byte, size) + if _, err := r.ReadAt(buf, offset); err != nil { + return nil, fmt.Errorf("reading region at %d: %w", offset, err) + } + if sum := sha1.Sum(buf); !bytes.Equal(sum[:], want[:]) { //nolint:gosec + return nil, fmt.Errorf("hash mismatch (corrupt or unsupported format)") + } + return buf, nil +} + +// Close releases the underlying file handle. +func (r *Reader) Close() error { return r.f.Close() } + +// Files returns every file this pak's index describes. +func (r *Reader) Files() []FileEntry { + out := make([]FileEntry, len(r.entries)) + for i, e := range r.entries { + out[i] = e.FileEntry + } + return out +} + +type footer struct { + version int32 + indexOffset int64 + indexSize int64 + indexHash [20]byte + encryptedIndex bool +} + +// readFooter parses the single 221-byte footer shape this package supports. +// The footer is fixed-size and sits flush against EOF, so there is nothing to +// search for and no alternate width to try: if Magic isn't where it must be, +// this is not a pak we handle. +func readFooter(r io.ReaderAt, fileSize int64) (footer, error) { + if fileSize < footerSize { + return footer{}, fmt.Errorf("%w: file of %d bytes is smaller than a %d-byte footer", + ErrUnsupportedFormat, fileSize, footerSize) + } + buf := make([]byte, footerSize) + if _, err := r.ReadAt(buf, fileSize-footerSize); err != nil { + return footer{}, fmt.Errorf("reading footer: %w", err) + } + // Layout: EncryptionKeyGuid(0:16) bEncryptedIndex(16) Magic(17:21) + // Version(21:25) IndexOffset(25:33) IndexSize(33:41) IndexHash(41:61) + // CompressionMethods(61:221). + if binary.LittleEndian.Uint32(buf[17:21]) != magic { + return footer{}, fmt.Errorf("%w: no pak magic at the expected footer offset", ErrUnsupportedFormat) + } + ft := footer{ + encryptedIndex: buf[16] != 0, + version: int32(binary.LittleEndian.Uint32(buf[21:25])), + indexOffset: int64(binary.LittleEndian.Uint64(buf[25:33])), + indexSize: int64(binary.LittleEndian.Uint64(buf[33:41])), + } + copy(ft.indexHash[:], buf[41:61]) + if ft.version < minVersion { + return footer{}, fmt.Errorf("%w: pak version %d (this package requires >= %d)", + ErrUnsupportedFormat, ft.version, minVersion) + } + // The trailing CompressionMethods name table is intentionally left + // unparsed: entries carry a method *index*, and this package only ever + // reads payloads whose index is 0 (stored), which needs no name. + return ft, nil +} + +// parseIndex parses the primary index, then the full directory index it points +// at, resolving every path to its bit-packed entry record. +// +// Version-11 paks have no flat entry array. The primary index holds a blob of +// bit-packed records plus SHA1-gated offsets to two sub-indexes: a path-hash +// index (hash -> record offset) and a full directory index +// (directory -> file -> record offset). Enumeration uses the directory index, +// which is the only one that carries real path strings. +func parseIndex(f io.ReaderAt, index []byte) ([]readerEntry, error) { + c := &cursor{b: index} + c.fstring() // MountPoint: recorded for the engine's benefit, unused here + numEntries := c.i32() + seed := c.u64() + _ = seed // only the writer needs the seed; enumeration goes via the directory index + + pathHash, err := readSubIndexRef(c, "path hash index") + if err != nil { + return nil, err + } + fullDir, err := readSubIndexRef(c, "full directory index") + if err != nil { + return nil, err + } + encoded := c.bytes(int(c.i32())) // EncodedPakEntriesSize, then the blob + if nonEncoded := c.i32(); nonEncoded != 0 { + return nil, fmt.Errorf("%w: %d non-encoded index entries", ErrUnsupportedFormat, nonEncoded) + } + if c.err != nil { + return nil, fmt.Errorf("primary index: %w", c.err) + } + + // Verify the path-hash index's hash even though enumeration does not use + // it: it is part of the format's integrity chain, and a pak whose + // sub-index hashes don't hold is not one to trust. + if _, err := readRegion(f, pathHash.offset, pathHash.size, pathHash.hash); err != nil { + return nil, fmt.Errorf("path hash index: %w", err) + } + dirBuf, err := readRegion(f, fullDir.offset, fullDir.size, fullDir.hash) + if err != nil { + return nil, fmt.Errorf("full directory index: %w", err) + } + + entries, err := parseDirectoryIndex(dirBuf, encoded) + if err != nil { + return nil, err + } + if int32(len(entries)) != numEntries { + return nil, fmt.Errorf("directory index lists %d files, index header says %d", + len(entries), numEntries) + } + sort.Slice(entries, func(i, j int) bool { return entries[i].Path < entries[j].Path }) + return entries, nil +} + +type subIndexRef struct { + offset, size int64 + hash [20]byte +} + +// readSubIndexRef reads a `bHasIndex` flag and, when set, the offset/size/ +// hash triple that follows. Both sub-indexes are required: every pak version +// this package accepts writes both, and a reader that limped along without the +// directory index would have no paths to report. +func readSubIndexRef(c *cursor, name string) (subIndexRef, error) { + if c.i32() == 0 { + return subIndexRef{}, fmt.Errorf("%w: pak has no %s", ErrUnsupportedFormat, name) + } + ref := subIndexRef{offset: c.i64(), size: c.i64()} + copy(ref.hash[:], c.bytes(20)) + return ref, c.err +} + +// parseDirectoryIndex walks directory -> file -> entry-location and decodes the +// bit-packed record each location points at. +func parseDirectoryIndex(dir, encoded []byte) ([]readerEntry, error) { + c := &cursor{b: dir} + dirCount := c.i32() + var entries []readerEntry + for i := int32(0); i < dirCount && c.err == nil; i++ { + dirName := c.fstring() + fileCount := c.i32() + for j := int32(0); j < fileCount && c.err == nil; j++ { + fileName := c.fstring() + loc := c.i32() + // Root-level files live under a "/" directory key, so the naive + // join yields a leading slash; the canonical mount-relative path + // (and the one hashPath consumes) has none. + full := strings.TrimPrefix(dirName+fileName, "/") + if loc < 0 { + // Negative locations index a non-encoded FPakEntry array. No + // pak in a real Icarus install uses them. + return nil, fmt.Errorf("entry %q: %w: non-encoded entry location", full, ErrUnsupportedFormat) + } + e, err := decodeEntry(encoded, int(loc)) + if err != nil { + return nil, fmt.Errorf("entry %q: %w", full, err) + } + e.Path = full + entries = append(entries, e) + } + } + if c.err != nil { + return nil, fmt.Errorf("directory index: %w", c.err) + } + return entries, nil +} + +// decodeEntry decodes one bit-packed FPakEntry from the encoded blob. +// +// The leading uint32 packs: bit31 offset-is-32-bit, bit30 uncompressed-size- +// is-32-bit, bit29 size-is-32-bit, bits28-23 CompressionMethodIndex, bit22 +// encrypted, bits21-6 compression block count, bits5-0 CompressionBlockSize>>11 +// (0x3f = escape, an explicit uint32 follows). Fields then appear in this +// order: [CompressionBlockSize] Offset, UncompressedSize, [Size], [block +// sizes]. Size is omitted for stored entries (it equals UncompressedSize), and +// the per-block size table is omitted for a lone unencrypted block. +// +// The block-size-before-Offset ordering is easy to get wrong; it was pinned +// down empirically and this decoder reproduces all 173,078 records across a +// real install exactly. See docs/plans/icarus-pak-format-findings.md. +func decodeEntry(b []byte, at int) (readerEntry, error) { + c := &cursor{b: b, pos: at} + flags := c.u32() + var ( + method = int32((flags >> 23) & 0x3F) + blockCount = int((flags >> 6) & 0xFFFF) + encrypted = flags&(1<<22) != 0 + ) + if flags&0x3F == 0x3F { + c.u32() // explicit CompressionBlockSize + } + read := func(is32 bool) int64 { + if is32 { + return int64(c.u32()) + } + return int64(c.u64()) + } + offset := read(flags&(1<<31) != 0) + uncompressed := read(flags&(1<<30) != 0) + if method != 0 { + read(flags&(1<<29) != 0) // Size on disk; unused, we refuse to read these payloads + } + if blockCount > 0 && (blockCount > 1 || encrypted) { + c.bytes(4 * blockCount) + } + if c.err != nil { + return readerEntry{}, fmt.Errorf("decoding entry at blob offset %d: %w", at, c.err) + } + if encrypted { + return readerEntry{}, fmt.Errorf("%w: encrypted entry", ErrUnsupportedFormat) + } + return readerEntry{ + FileEntry: FileEntry{Size: uncompressed}, + offset: offset, + method: method, + }, nil +} + +// cursor is a bounds-checked little-endian cursor over an in-memory index +// region. It latches the first error so parse code can read a whole structure +// and check once, rather than wrapping every field. +type cursor struct { + b []byte + pos int + err error +} + +func (c *cursor) take(n int) []byte { + if c.err != nil { + return make([]byte, n) + } + if n < 0 || c.pos+n > len(c.b) { + c.err = io.ErrUnexpectedEOF + return make([]byte, max(n, 0)) + } + v := c.b[c.pos : c.pos+n] + c.pos += n + return v +} + +func (c *cursor) bytes(n int) []byte { return c.take(n) } +func (c *cursor) u32() uint32 { return binary.LittleEndian.Uint32(c.take(4)) } +func (c *cursor) i32() int32 { return int32(c.u32()) } +func (c *cursor) u64() uint64 { return binary.LittleEndian.Uint64(c.take(8)) } +func (c *cursor) i64() int64 { return int64(c.u64()) } + +// fstring reads a length-prefixed Unreal FString. A negative length signals +// UTF-16, which no pak in a real Icarus install uses and this package does not +// decode. +func (c *cursor) fstring() string { + n := c.i32() + if n == 0 || c.err != nil { + return "" + } + if n < 0 { + c.err = fmt.Errorf("%w: UTF-16 FString", ErrUnsupportedFormat) + return "" + } + return string(bytes.TrimRight(c.take(int(n)), "\x00")) +} diff --git a/internal/unrealpak/reader_test.go b/internal/unrealpak/reader_test.go new file mode 100644 index 0000000..49484fd --- /dev/null +++ b/internal/unrealpak/reader_test.go @@ -0,0 +1,189 @@ +package unrealpak + +import ( + "bytes" + "crypto/sha1" //nolint:gosec // pak format uses SHA1, not our choice + "encoding/binary" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +// fixtureSeed is an arbitrary PathHashSeed for fixture paks. Readers take the +// seed from the index, so any value works — real paks use a different one per +// chunk. +const fixtureSeed uint64 = 0x0123456789ABCDEF + +// writeMinimalPak builds a hand-crafted but fully valid version-11 pak holding +// a single stored entry: data section, primary index, path-hash index, full +// directory index, then the 221-byte footer. It deliberately does not use the +// Task 4 Writer — the reader's tests must be able to fail independently of the +// writer, and vice versa. +func writeMinimalPak(t *testing.T, mountPath string, content []byte) string { + t.Helper() + return writeMinimalPakMethod(t, mountPath, content, 0) +} + +// writeMinimalPakMethod builds a fixture whose entry claims CompressionMethodIndex +// method. Only method 0 produces a genuinely readable pak; non-zero values exist +// to exercise the reader's refusal path (Task 3), which is the case that matters +// in practice — 74% of real Icarus entries are Oodle-compressed. +func writeMinimalPakMethod(t *testing.T, mountPath string, content []byte, method int32) string { + t.Helper() + pakPath := filepath.Join(t.TempDir(), "test.pak") + if err := os.WriteFile(pakPath, buildFixturePak(mountPath, content, method), 0o644); err != nil { + t.Fatalf("writing test pak: %v", err) + } + return pakPath +} + +func buildFixturePak(mountPath string, content []byte, method int32) []byte { + rel := strings.TrimPrefix(mountPath, "/") + + // Data section: the 53-byte per-entry header, then the payload, at offset 0. + var data bytes.Buffer + hdr := storedEntryHeader(int64(len(content)), content) + binary.LittleEndian.PutUint32(hdr[24:28], uint32(method)) // CompressionMethodIndex + data.Write(hdr) + data.Write(content) + + // One encoded index record. For method 0 that is the 12-byte stored shape: + // flags 0xE0000000 (offset/uncompressed-size/size all 32-bit-safe, no + // blocks), then uint32 Offset and uint32 UncompressedSize — Size is not + // serialized for method 0, it equals UncompressedSize. A non-zero method + // adds the uint32 Size field, per the encoded-record layout. + var encoded bytes.Buffer + binary.Write(&encoded, binary.LittleEndian, uint32(0xE0000000)|uint32(method)<<23) //nolint:errcheck + binary.Write(&encoded, binary.LittleEndian, uint32(0)) //nolint:errcheck + binary.Write(&encoded, binary.LittleEndian, uint32(len(content))) //nolint:errcheck + if method != 0 { + binary.Write(&encoded, binary.LittleEndian, uint32(len(content))) //nolint:errcheck // Size + } + + // Full directory index: one directory, one file, pointing at blob offset 0. + dirName, fileName := splitMountPath(rel) + var fdi bytes.Buffer + binary.Write(&fdi, binary.LittleEndian, int32(1)) //nolint:errcheck // DirCount + writeFString(&fdi, dirName) + binary.Write(&fdi, binary.LittleEndian, int32(1)) //nolint:errcheck // FileCount + writeFString(&fdi, fileName) + binary.Write(&fdi, binary.LittleEndian, int32(0)) //nolint:errcheck // PakEntryLocation + + // Path-hash index: the hash->location map, then an EMPTY pruned directory + // index. 33 of the 34 paks in a real install ship it empty, so a bare + // int32(0) is a shape the engine demonstrably accepts. + var phi bytes.Buffer + binary.Write(&phi, binary.LittleEndian, int32(1)) //nolint:errcheck // Count + binary.Write(&phi, binary.LittleEndian, hashPath(rel, fixtureSeed)) //nolint:errcheck + binary.Write(&phi, binary.LittleEndian, int32(0)) //nolint:errcheck // location + binary.Write(&phi, binary.LittleEndian, int32(0)) //nolint:errcheck // pruned index: 0 dirs + + phiHash := sha1.Sum(phi.Bytes()) //nolint:gosec + fdiHash := sha1.Sum(fdi.Bytes()) //nolint:gosec + + indexOffset := int64(data.Len()) + sizing := buildPrimaryIndex(1, fixtureSeed, 0, 0, phiHash, 0, 0, fdiHash, encoded.Bytes()) + phiOffset := indexOffset + int64(len(sizing)) + fdiOffset := phiOffset + int64(phi.Len()) + index := buildPrimaryIndex(1, fixtureSeed, + phiOffset, int64(phi.Len()), phiHash, + fdiOffset, int64(fdi.Len()), fdiHash, encoded.Bytes()) + indexHash := sha1.Sum(index) //nolint:gosec + + var out bytes.Buffer + out.Write(data.Bytes()) + out.Write(index) + out.Write(phi.Bytes()) + out.Write(fdi.Bytes()) + out.Write(buildFooter(writeVersion, indexOffset, int64(len(index)), indexHash)) + return out.Bytes() +} + +func TestReader_Open_ListsFiles(t *testing.T) { + content := []byte(`{"hello":"world"}`) + path := writeMinimalPak(t, "Icarus/Content/Data/Test.json", content) + + r, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() + + files := r.Files() + if len(files) != 1 { + t.Fatalf("got %d files, want 1", len(files)) + } + if files[0].Path != "Icarus/Content/Data/Test.json" { + t.Errorf("Path = %q, want Icarus/Content/Data/Test.json", files[0].Path) + } + if files[0].Size != int64(len(content)) { + t.Errorf("Size = %d, want %d", files[0].Size, len(content)) + } +} + +// A root-level file is keyed under the "/" directory in the directory index; +// Files must report it without the leading slash, matching what hashPath uses. +func TestReader_Open_RootLevelFile(t *testing.T) { + path := writeMinimalPak(t, "x.json", []byte("{}")) + r, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() + + files := r.Files() + if len(files) != 1 || files[0].Path != "x.json" { + t.Fatalf("Files() = %+v, want one entry with Path %q", files, "x.json") + } +} + +func TestReader_Open_RejectsEncryptedIndex(t *testing.T) { + path := writeMinimalPak(t, "x.json", []byte("{}")) + data, _ := os.ReadFile(path) + // bEncryptedIndex sits at offset 16 from footer start — right after the + // 16-byte EncryptionKeyGuid, immediately before Magic. + data[len(data)-footerSize+16] = 1 + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatal(err) + } + + _, err := Open(path) + if !errors.Is(err, ErrUnsupportedFormat) { + t.Fatalf("Open error = %v, want ErrUnsupportedFormat", err) + } +} + +// Versions below 10 use a flat index this package deliberately does not parse: +// a hard error, never a fallback. +func TestReader_Open_RejectsPreVersion10(t *testing.T) { + path := writeMinimalPak(t, "x.json", []byte("{}")) + data, _ := os.ReadFile(path) + // Version is the int32 at footer offset 21 (after Guid+flag+Magic). + binary.LittleEndian.PutUint32(data[len(data)-footerSize+21:], uint32(9)) + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatal(err) + } + + _, err := Open(path) + if !errors.Is(err, ErrUnsupportedFormat) { + t.Fatalf("Open error = %v, want ErrUnsupportedFormat", err) + } +} + +// Corruption anywhere in the index must trip a SHA1 gate rather than be +// parsed. The full directory index is the last region before the footer, so +// flipping the byte just before it exercises the primary->sub-index gate. +func TestReader_Open_RejectsCorruptedDirectoryIndex(t *testing.T) { + path := writeMinimalPak(t, "x.json", []byte("{}")) + data, _ := os.ReadFile(path) + data[len(data)-footerSize-1] ^= 0xFF + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatal(err) + } + + if _, err := Open(path); err == nil { + t.Fatal("expected error for corrupted directory index, got nil") + } +} From b4726851b05824ee5eb59db96924e5e2443e5a1f Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Fri, 31 Jul 2026 19:30:50 -0400 Subject: [PATCH 02/96] feat: add unrealpak file content reading (#136) --- internal/unrealpak/reader.go | 45 +++++++++++++++++++++++++++++++ internal/unrealpak/reader_test.go | 42 +++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/internal/unrealpak/reader.go b/internal/unrealpak/reader.go index 2d1bddc..e2c2c2e 100644 --- a/internal/unrealpak/reader.go +++ b/internal/unrealpak/reader.go @@ -93,6 +93,51 @@ func (r *Reader) Files() []FileEntry { return out } +// ReadFile returns the bytes of the entry at mount-relative path. +// +// On-disk entry data is preceded by a full FPakEntry header — 53 bytes for a +// stored entry (Offset, Size, UncompressedSize, CompressionMethodIndex, Hash, +// Flags, CompressionBlockSize) — and the index's offset points at that header, +// not the payload. The header is re-read and cross-checked rather than trusted: +// its method and size must agree with the index, and its Hash must match the +// payload's SHA1. Real paks satisfy all three (verified across a whole install), +// so a disagreement means corruption or a layout this package misread. +func (r *Reader) ReadFile(path string) ([]byte, error) { + for _, e := range r.entries { + if e.Path != path { + continue + } + // Compression is refused here rather than at index-parse time so that + // Files() can still enumerate real paks, most of whose entries are + // Oodle-compressed. No caller can obtain wrong bytes either way. + if e.method != 0 { + return nil, fmt.Errorf("unrealpak: %s: %w: compressed entry (method %d)", + path, ErrUnsupportedFormat, e.method) + } + hdr := make([]byte, storedHeaderSize) + if _, err := r.f.ReadAt(hdr, e.offset); err != nil { + return nil, fmt.Errorf("unrealpak: %s: reading entry header: %w", path, err) + } + if m := int32(binary.LittleEndian.Uint32(hdr[24:28])); m != 0 { + return nil, fmt.Errorf("unrealpak: %s: %w: compressed entry data (method %d)", + path, ErrUnsupportedFormat, m) + } + if size := int64(binary.LittleEndian.Uint64(hdr[8:16])); size != e.Size { + return nil, fmt.Errorf("unrealpak: %s: entry header size %d disagrees with index size %d", + path, size, e.Size) + } + buf := make([]byte, e.Size) + if _, err := r.f.ReadAt(buf, e.offset+storedHeaderSize); err != nil { + return nil, fmt.Errorf("unrealpak: reading %s: %w", path, err) + } + if sum := sha1.Sum(buf); !bytes.Equal(sum[:], hdr[28:48]) { //nolint:gosec + return nil, fmt.Errorf("unrealpak: %s: content hash mismatch", path) + } + return buf, nil + } + return nil, fmt.Errorf("unrealpak: %s: %w", path, os.ErrNotExist) +} + type footer struct { version int32 indexOffset int64 diff --git a/internal/unrealpak/reader_test.go b/internal/unrealpak/reader_test.go index 49484fd..fedf7f4 100644 --- a/internal/unrealpak/reader_test.go +++ b/internal/unrealpak/reader_test.go @@ -187,3 +187,45 @@ func TestReader_Open_RejectsCorruptedDirectoryIndex(t *testing.T) { t.Fatal("expected error for corrupted directory index, got nil") } } + +func TestReader_ReadFile(t *testing.T) { + content := []byte(`{"hello":"world"}`) + path := writeMinimalPak(t, "Icarus/Content/Data/Test.json", content) + + r, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() + + got, err := r.ReadFile("Icarus/Content/Data/Test.json") + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(got) != string(content) { + t.Errorf("got %q, want %q", got, content) + } + + if _, err := r.ReadFile("does/not/exist.json"); err == nil { + t.Error("expected error for missing file, got nil") + } +} + +func TestReader_ReadFile_RejectsCompressedEntry(t *testing.T) { + const name = "Items/D_ItemsStatic.json" + path := writeMinimalPakMethod(t, name, []byte(`{"a":1}`), 1) // 1 = Oodle + + r, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() + + // Enumeration must still work — the reader lists compressed entries. + if files := r.Files(); len(files) != 1 || files[0].Path != name { + t.Fatalf("Files() = %+v, want one entry named %q", files, name) + } + if _, err := r.ReadFile(name); !errors.Is(err, ErrUnsupportedFormat) { + t.Fatalf("ReadFile error = %v, want ErrUnsupportedFormat", err) + } +} From eab7e103c201f273e36184aa6811bea6828362e2 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Fri, 31 Jul 2026 19:35:32 -0400 Subject: [PATCH 03/96] feat: add unrealpak writer (#136) --- internal/unrealpak/writer.go | 162 ++++++++++++++++++++++++++++++ internal/unrealpak/writer_test.go | 100 ++++++++++++++++++ 2 files changed, 262 insertions(+) create mode 100644 internal/unrealpak/writer.go create mode 100644 internal/unrealpak/writer_test.go diff --git a/internal/unrealpak/writer.go b/internal/unrealpak/writer.go new file mode 100644 index 0000000..31f9be6 --- /dev/null +++ b/internal/unrealpak/writer.go @@ -0,0 +1,162 @@ +package unrealpak + +import ( + "bytes" + "crypto/sha1" //nolint:gosec // pak format uses SHA1, not our choice + "encoding/binary" + "fmt" + "math" + "os" + "slices" + "sort" + "strings" +) + +// writerSeed is the PathHashSeed stamped into written paks. Any value works — +// readers take the seed from the index, and real paks use a different one per +// chunk — but a fixed one keeps output deterministic. +const writerSeed uint64 = 0x9E3779B97F4A7C15 + +// Writer produces a stored (uncompressed), unencrypted version-11 pak carrying +// the full three-part index: primary index, path-hash index and full directory +// index, then the 221-byte footer. +// +// AddFile buffers content in memory and Close emits everything sorted by path, +// so identical inputs produce byte-identical output regardless of AddFile call +// order. Mod paks are small — Icarus's entire base data.pak is 2.4 MB — so +// buffering costs little, and deterministic output is worth more: it makes the +// round-trip test able to assert on bytes and keeps compiled paks stable across +// recompiles. +type Writer struct { + f *os.File + closed bool + files []writerFile + seen map[string]bool +} + +type writerFile struct { + path string + data []byte +} + +// Create opens path for writing. Call AddFile for each entry, then Close. +func Create(path string) (*Writer, error) { + f, err := os.Create(path) + if err != nil { + return nil, fmt.Errorf("unrealpak: creating %s: %w", path, err) + } + return &Writer{f: f, seen: make(map[string]bool)}, nil +} + +// AddFile records one entry. Nothing reaches disk until Close. +func (w *Writer) AddFile(mountPath string, data []byte) error { + if w.closed { + return fmt.Errorf("unrealpak: AddFile on closed writer") + } + // Root-level files are keyed under "/" in the directory index, but the + // canonical path — and the one hashPath consumes — carries no leading slash. + rel := strings.TrimPrefix(mountPath, "/") + if rel == "" { + return fmt.Errorf("unrealpak: AddFile: empty mount path") + } + if w.seen[rel] { + return fmt.Errorf("unrealpak: AddFile: duplicate path %q", rel) + } + w.seen[rel] = true + w.files = append(w.files, writerFile{path: rel, data: slices.Clone(data)}) + return nil +} + +// Close assembles the data section and all three index structures, writes them +// with the footer, and closes the file. +func (w *Writer) Close() error { + if w.closed { + return nil + } + w.closed = true + + sort.Slice(w.files, func(i, j int) bool { return w.files[i].path < w.files[j].path }) + + // Data section and encoded index records, in one pass. Each payload is + // preceded by its 53-byte header; entries are packed with no padding. + var data, encoded bytes.Buffer + locations := make(map[string]int32, len(w.files)) + for _, file := range w.files { + offset, size := int64(data.Len()), int64(len(file.data)) + if offset > math.MaxUint32 || size > math.MaxUint32 { + w.f.Close() //nolint:errcheck + return fmt.Errorf("unrealpak: %s: offset/size exceeds the 32-bit encoded-entry form this writer emits", file.path) + } + data.Write(storedEntryHeader(size, file.data)) + data.Write(file.data) + + locations[file.path] = int32(encoded.Len()) + // The 12-byte stored record: offset/uncompressed-size/size all + // 32-bit-safe, method 0, no compression blocks. + binary.Write(&encoded, binary.LittleEndian, uint32(0xE0000000)) //nolint:errcheck + binary.Write(&encoded, binary.LittleEndian, uint32(offset)) //nolint:errcheck + binary.Write(&encoded, binary.LittleEndian, uint32(size)) //nolint:errcheck + } + + // Full directory index: directory -> file -> encoded-record location. + byDir := make(map[string][]string) + for _, file := range w.files { + dir, name := splitMountPath(file.path) + byDir[dir] = append(byDir[dir], name) + } + dirNames := make([]string, 0, len(byDir)) + for dir := range byDir { + dirNames = append(dirNames, dir) + } + sort.Strings(dirNames) + + var fdi bytes.Buffer + binary.Write(&fdi, binary.LittleEndian, int32(len(dirNames))) //nolint:errcheck + for _, dir := range dirNames { + writeFString(&fdi, dir) + names := byDir[dir] + sort.Strings(names) + binary.Write(&fdi, binary.LittleEndian, int32(len(names))) //nolint:errcheck + for _, name := range names { + writeFString(&fdi, name) + binary.Write(&fdi, binary.LittleEndian, locations[strings.TrimPrefix(dir+name, "/")]) //nolint:errcheck + } + } + + // Path-hash index, then an empty pruned directory index. + var phi bytes.Buffer + binary.Write(&phi, binary.LittleEndian, int32(len(w.files))) //nolint:errcheck + for _, file := range w.files { + binary.Write(&phi, binary.LittleEndian, hashPath(file.path, writerSeed)) //nolint:errcheck + binary.Write(&phi, binary.LittleEndian, locations[file.path]) //nolint:errcheck + } + binary.Write(&phi, binary.LittleEndian, int32(0)) //nolint:errcheck // pruned index: 0 directories + + // The primary index records absolute offsets of the two sub-indexes that + // follow it; its own length is independent of those values, so measure it + // with zeros first, then rebuild with the real offsets. + phiHash := sha1.Sum(phi.Bytes()) //nolint:gosec + fdiHash := sha1.Sum(fdi.Bytes()) //nolint:gosec + count := int32(len(w.files)) + indexOffset := int64(data.Len()) + sizing := buildPrimaryIndex(count, writerSeed, 0, 0, phiHash, 0, 0, fdiHash, encoded.Bytes()) + phiOffset := indexOffset + int64(len(sizing)) + fdiOffset := phiOffset + int64(phi.Len()) + index := buildPrimaryIndex(count, writerSeed, + phiOffset, int64(phi.Len()), phiHash, + fdiOffset, int64(fdi.Len()), fdiHash, encoded.Bytes()) + indexHash := sha1.Sum(index) //nolint:gosec + + // Regions tile the file exactly, as they do in every real pak: + // data | primary index | path-hash index | full directory index | footer. + for _, chunk := range [][]byte{ + data.Bytes(), index, phi.Bytes(), fdi.Bytes(), + buildFooter(writeVersion, indexOffset, int64(len(index)), indexHash), + } { + if _, err := w.f.Write(chunk); err != nil { + w.f.Close() //nolint:errcheck + return fmt.Errorf("unrealpak: writing pak: %w", err) + } + } + return w.f.Close() +} diff --git a/internal/unrealpak/writer_test.go b/internal/unrealpak/writer_test.go new file mode 100644 index 0000000..7a7d1a6 --- /dev/null +++ b/internal/unrealpak/writer_test.go @@ -0,0 +1,100 @@ +package unrealpak + +import ( + "bytes" + "encoding/binary" + "os" + "path/filepath" + "testing" +) + +func TestWriter_CreateAndClose_ProducesValidFooter(t *testing.T) { + path := filepath.Join(t.TempDir(), "out.pak") + w, err := Create(path) + if err != nil { + t.Fatalf("Create: %v", err) + } + if err := w.AddFile("Icarus/Content/Data/Test.json", []byte(`{"a":1}`)); err != nil { + t.Fatalf("AddFile: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("reading output: %v", err) + } + if len(data) <= footerSize { + t.Fatalf("output is %d bytes, want more than a bare %d-byte footer", len(data), footerSize) + } + ft := data[len(data)-footerSize:] + if got := binary.LittleEndian.Uint32(ft[17:21]); got != magic { + t.Errorf("footer magic = %#x, want %#x", got, magic) + } + if got := int32(binary.LittleEndian.Uint32(ft[21:25])); got != writeVersion { + t.Errorf("footer version = %d, want %d", got, writeVersion) + } + if ft[16] != 0 { + t.Errorf("bEncryptedIndex = %d, want 0", ft[16]) + } +} + +// Output must not depend on AddFile ordering — Close sorts by path. +func TestWriter_Close_IsDeterministic(t *testing.T) { + build := func(order []string) []byte { + t.Helper() + path := filepath.Join(t.TempDir(), "out.pak") + w, err := Create(path) + if err != nil { + t.Fatalf("Create: %v", err) + } + for _, name := range order { + if err := w.AddFile(name, []byte(name)); err != nil { + t.Fatalf("AddFile(%s): %v", name, err) + } + } + if err := w.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("reading output: %v", err) + } + return data + } + + a := build([]string{"a/one.json", "b/two.json", "root.json"}) + b := build([]string{"root.json", "b/two.json", "a/one.json"}) + if !bytes.Equal(a, b) { + t.Error("output differs with AddFile order; Close must be deterministic") + } +} + +func TestWriter_AddFile_RejectsDuplicatePath(t *testing.T) { + w, err := Create(filepath.Join(t.TempDir(), "out.pak")) + if err != nil { + t.Fatalf("Create: %v", err) + } + defer w.Close() //nolint:errcheck + if err := w.AddFile("x.json", []byte("{}")); err != nil { + t.Fatalf("AddFile: %v", err) + } + if err := w.AddFile("x.json", []byte("{}")); err == nil { + t.Error("expected error adding a duplicate path, got nil") + } +} + +func TestWriter_AddFile_AfterClose_Errors(t *testing.T) { + path := filepath.Join(t.TempDir(), "out.pak") + w, err := Create(path) + if err != nil { + t.Fatalf("Create: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + if err := w.AddFile("x.json", []byte("{}")); err == nil { + t.Error("expected error adding file after Close, got nil") + } +} From 70e28382038e730cdf44753a0eee142a13f3f84a Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Fri, 31 Jul 2026 19:40:44 -0400 Subject: [PATCH 04/96] test: add unrealpak writer/reader round-trip coverage (#136) --- internal/unrealpak/roundtrip_test.go | 100 +++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 internal/unrealpak/roundtrip_test.go diff --git a/internal/unrealpak/roundtrip_test.go b/internal/unrealpak/roundtrip_test.go new file mode 100644 index 0000000..2d0b9db --- /dev/null +++ b/internal/unrealpak/roundtrip_test.go @@ -0,0 +1,100 @@ +package unrealpak + +import ( + "crypto/sha1" //nolint:gosec // pak format uses SHA1, not our choice + "encoding/binary" + "os" + "path/filepath" + "testing" +) + +func TestRoundTrip_WriteThenRead(t *testing.T) { + path := filepath.Join(t.TempDir(), "roundtrip.pak") + files := map[string][]byte{ + "Icarus/Content/Data/AI-D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":235}}`), + "Icarus/Content/Data/Other.json": []byte(`{"foo":"bar"}`), + "DataTableMetadata.json": []byte(`{"root":true}`), // root-level: "/" directory key + } + + w, err := Create(path) + if err != nil { + t.Fatalf("Create: %v", err) + } + for name, data := range files { + if err := w.AddFile(name, data); err != nil { + t.Fatalf("AddFile(%s): %v", name, err) + } + } + if err := w.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + r, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() + + got := r.Files() + if len(got) != len(files) { + t.Fatalf("got %d files, want %d", len(got), len(files)) + } + for name, want := range files { + data, err := r.ReadFile(name) + if err != nil { + t.Fatalf("ReadFile(%s): %v", name, err) + } + if string(data) != string(want) { + t.Errorf("ReadFile(%s) = %q, want %q", name, data, want) + } + } +} + +// Structural assertions on the bytes themselves. Open() proves the Reader +// accepts what the Writer emits, but the Reader is not the audience that +// matters most — Icarus's engine is. These check the properties every real pak +// exhibits, so a drift away from the engine-proven shape fails here rather than +// silently in-game. +func TestRoundTrip_StructuralShape(t *testing.T) { + path := filepath.Join(t.TempDir(), "shape.pak") + w, err := Create(path) + if err != nil { + t.Fatalf("Create: %v", err) + } + content := []byte(`{"a":1}`) + if err := w.AddFile("Icarus/Content/Data/Test.json", content); err != nil { + t.Fatalf("AddFile: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("reading output: %v", err) + } + ft := data[len(data)-footerSize:] + indexOffset := int64(binary.LittleEndian.Uint64(ft[25:33])) + indexSize := int64(binary.LittleEndian.Uint64(ft[33:41])) + + // The footer's SHA1 must cover the primary index exactly. + indexSum := sha1.Sum(data[indexOffset : indexOffset+indexSize]) //nolint:gosec + if string(indexSum[:]) != string(ft[41:61]) { + t.Error("footer IndexHash does not match the primary index bytes") + } + + // The data section holds one 53-byte header plus the payload, and the + // index starts immediately after it — regions tile with no gap. + if want := int64(storedHeaderSize + len(content)); indexOffset != want { + t.Errorf("index starts at %d, want %d (53-byte header + %d-byte payload)", + indexOffset, want, len(content)) + } + // The per-entry header's Hash field must be the payload's SHA1. + if sum := sha1.Sum(content); string(sum[:]) != string(data[28:48]) { //nolint:gosec + t.Error("per-entry header Hash does not match the payload SHA1") + } + // Its Offset field is zero, as in every real pak. + if got := binary.LittleEndian.Uint64(data[0:8]); got != 0 { + t.Errorf("per-entry header Offset = %d, want 0", got) + } +} From d524f642ece28f07ba0f31a720c21ebf586db2ab Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Fri, 31 Jul 2026 19:44:56 -0400 Subject: [PATCH 05/96] feat: add Firestore typed-value decoder for Icarus source (#136) --- internal/source/icarus/firestore_value.go | 51 +++++++++++++++++++ .../source/icarus/firestore_value_test.go | 34 +++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 internal/source/icarus/firestore_value.go create mode 100644 internal/source/icarus/firestore_value_test.go diff --git a/internal/source/icarus/firestore_value.go b/internal/source/icarus/firestore_value.go new file mode 100644 index 0000000..18f0a4b --- /dev/null +++ b/internal/source/icarus/firestore_value.go @@ -0,0 +1,51 @@ +package icarus + +// decodeFields unwraps a Firestore REST document's typed-value "fields" +// object (each value wrapped as {"stringValue": ...} / {"mapValue": {...}} / +// etc.) into plain Go values. Only the value kinds this catalog's schema +// actually uses are handled; anything else decodes to nil rather than +// panicking, since an unrecognized field should be ignorable, not fatal. +func decodeFields(fields map[string]any) map[string]any { + out := make(map[string]any, len(fields)) + for k, v := range fields { + out[k] = decodeValue(v) + } + return out +} + +func decodeValue(v any) any { + wrapped, ok := v.(map[string]any) + if !ok { + return nil + } + if s, ok := wrapped["stringValue"]; ok { + return s + } + if b, ok := wrapped["booleanValue"]; ok { + return b + } + if i, ok := wrapped["integerValue"]; ok { + return i + } + if d, ok := wrapped["doubleValue"]; ok { + return d + } + if m, ok := wrapped["mapValue"]; ok { + mv, _ := m.(map[string]any) + inner, _ := mv["fields"].(map[string]any) + return decodeFields(inner) + } + if a, ok := wrapped["arrayValue"]; ok { + av, _ := a.(map[string]any) + values, _ := av["values"].([]any) + out := make([]any, len(values)) + for i, item := range values { + out[i] = decodeValue(item) + } + return out + } + if _, ok := wrapped["nullValue"]; ok { + return nil + } + return nil +} diff --git a/internal/source/icarus/firestore_value_test.go b/internal/source/icarus/firestore_value_test.go new file mode 100644 index 0000000..47882e3 --- /dev/null +++ b/internal/source/icarus/firestore_value_test.go @@ -0,0 +1,34 @@ +package icarus + +import ( + "reflect" + "testing" +) + +func TestDecodeFields(t *testing.T) { + // Shape of a real Firestore REST document's "fields" object. + raw := map[string]any{ + "name": map[string]any{"stringValue": "Bear Mount"}, + "version": map[string]any{"stringValue": "3.3"}, + "files": map[string]any{"mapValue": map[string]any{"fields": map[string]any{ + "pak": map[string]any{"stringValue": "https://example.com/mod.pak"}, + "exmodz": map[string]any{"stringValue": "https://example.com/mod.exmodz"}, + }}}, + "missing": map[string]any{"nullValue": nil}, + } + + got := decodeFields(raw) + + want := map[string]any{ + "name": "Bear Mount", + "version": "3.3", + "files": map[string]any{ + "pak": "https://example.com/mod.pak", + "exmodz": "https://example.com/mod.exmodz", + }, + "missing": nil, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("decodeFields() = %#v, want %#v", got, want) + } +} From 24c9b51ae0e89c748f70009001043bd925b61c78 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Fri, 31 Jul 2026 19:48:19 -0400 Subject: [PATCH 06/96] feat: add Firestore REST client for Icarus source (#136) --- internal/source/icarus/firestore_client.go | 102 ++++++++++++++++++ .../source/icarus/firestore_client_test.go | 67 ++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 internal/source/icarus/firestore_client.go create mode 100644 internal/source/icarus/firestore_client_test.go diff --git a/internal/source/icarus/firestore_client.go b/internal/source/icarus/firestore_client.go new file mode 100644 index 0000000..07a7e6d --- /dev/null +++ b/internal/source/icarus/firestore_client.go @@ -0,0 +1,102 @@ +package icarus + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" +) + +const defaultFirestoreBaseURL = "https://firestore.googleapis.com/v1" + +// firestoreDoc is a decoded Firestore document: ID is the last path segment +// of its resource name, Fields is already unwrapped via decodeFields. +type firestoreDoc struct { + ID string + Fields map[string]any +} + +type firestoreClient struct { + projectID string + httpClient *http.Client + baseURL string // overridable in tests; defaults to defaultFirestoreBaseURL +} + +func newFirestoreClient(projectID string, httpClient *http.Client) *firestoreClient { + if httpClient == nil { + httpClient = http.DefaultClient + } + return &firestoreClient{projectID: projectID, httpClient: httpClient, baseURL: defaultFirestoreBaseURL} +} + +func (c *firestoreClient) documentsURL() string { + return fmt.Sprintf("%s/projects/%s/databases/(default)/documents", c.baseURL, c.projectID) +} + +// listCollection fetches every document in collection, following +// nextPageToken until exhausted (the catalog reads Firestore unauthenticated +// and public, with no server-side query support in play — see the design +// doc's "fetch-all + filter client-side" decision). +func (c *firestoreClient) listCollection(ctx context.Context, collection string) ([]firestoreDoc, error) { + var all []firestoreDoc + pageToken := "" + for { + url := fmt.Sprintf("%s/%s?pageSize=200", c.documentsURL(), collection) + if pageToken != "" { + url += "&pageToken=" + pageToken + } + var page struct { + Documents []struct { + Name string `json:"name"` + Fields map[string]any `json:"fields"` + } `json:"documents"` + NextPageToken string `json:"nextPageToken"` + } + if err := c.getJSON(ctx, url, &page); err != nil { + return nil, fmt.Errorf("listing %s: %w", collection, err) + } + for _, d := range page.Documents { + all = append(all, firestoreDoc{ID: lastPathSegment(d.Name), Fields: decodeFields(d.Fields)}) + } + if page.NextPageToken == "" { + break + } + pageToken = page.NextPageToken + } + return all, nil +} + +// getDocument fetches a single document by ID. +func (c *firestoreClient) getDocument(ctx context.Context, collection, docID string) (*firestoreDoc, error) { + url := fmt.Sprintf("%s/%s/%s", c.documentsURL(), collection, docID) + var doc struct { + Name string `json:"name"` + Fields map[string]any `json:"fields"` + } + if err := c.getJSON(ctx, url, &doc); err != nil { + return nil, fmt.Errorf("fetching %s/%s: %w", collection, docID, err) + } + return &firestoreDoc{ID: lastPathSegment(doc.Name), Fields: decodeFields(doc.Fields)}, nil +} + +func (c *firestoreClient) getJSON(ctx context.Context, url string, out any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + resp, err := c.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() //nolint:errcheck + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("HTTP %d", resp.StatusCode) + } + return json.NewDecoder(resp.Body).Decode(out) +} + +func lastPathSegment(resourceName string) string { + parts := strings.Split(resourceName, "/") + return parts[len(parts)-1] +} diff --git a/internal/source/icarus/firestore_client_test.go b/internal/source/icarus/firestore_client_test.go new file mode 100644 index 0000000..4fa5cec --- /dev/null +++ b/internal/source/icarus/firestore_client_test.go @@ -0,0 +1,67 @@ +package icarus + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestFirestoreClient_ListCollection_Paginates(t *testing.T) { + pages := []map[string]any{ + { + "documents": []map[string]any{ + {"name": "projects/p/databases/(default)/documents/mods/abc", "fields": map[string]any{"name": map[string]any{"stringValue": "Bear Mount"}}}, + }, + "nextPageToken": "page2", + }, + { + "documents": []map[string]any{ + {"name": "projects/p/databases/(default)/documents/mods/def", "fields": map[string]any{"name": map[string]any{"stringValue": "Wolf Mount"}}}, + }, + }, + } + callCount := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + page := pages[callCount] + callCount++ + json.NewEncoder(w).Encode(page) //nolint:errcheck + })) + defer srv.Close() + + c := newFirestoreClient("test-project", srv.Client()) + c.baseURL = srv.URL // test seam, see Step 3 + + docs, err := c.listCollection(context.Background(), "mods") + if err != nil { + t.Fatalf("listCollection: %v", err) + } + if len(docs) != 2 { + t.Fatalf("got %d docs, want 2 (pagination should have followed nextPageToken)", len(docs)) + } + if docs[0].ID != "abc" || docs[1].ID != "def" { + t.Errorf("doc IDs = %q, %q, want abc, def", docs[0].ID, docs[1].ID) + } + if docs[0].Fields["name"] != "Bear Mount" { + t.Errorf("docs[0].Fields[name] = %v, want Bear Mount", docs[0].Fields["name"]) + } + if callCount != 2 { + t.Errorf("callCount = %d, want 2 (one per page)", callCount) + } +} + +func TestFirestoreClient_GetDocument_NotFound(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + c := newFirestoreClient("test-project", srv.Client()) + c.baseURL = srv.URL + + _, err := c.getDocument(context.Background(), "mods", "missing") + if err == nil { + t.Fatal("expected error for 404, got nil") + } +} From a673cea5a9c6087ca7391315a6a00d921765b24f Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Fri, 31 Jul 2026 19:54:04 -0400 Subject: [PATCH 07/96] feat: implement Icarus ModSource over Firestore REST (#136) --- internal/source/icarus/icarus.go | 209 ++++++++++++++++++++++++++ internal/source/icarus/icarus_test.go | 85 +++++++++++ 2 files changed, 294 insertions(+) create mode 100644 internal/source/icarus/icarus.go create mode 100644 internal/source/icarus/icarus_test.go diff --git a/internal/source/icarus/icarus.go b/internal/source/icarus/icarus.go new file mode 100644 index 0000000..794fddb --- /dev/null +++ b/internal/source/icarus/icarus.go @@ -0,0 +1,209 @@ +package icarus + +import ( + "context" + "fmt" + "net/http" + "net/url" + "path" + "strings" + + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/source" +) + +// gameID is fixed: the Firestore database this source reads is Icarus-only. +const gameID = "icarus" + +// Icarus is a ModSource backed by the public, unauthenticated Firestore REST +// API described in docs/plans/2026-07-29-icarus-exmod-pak-research.md. +type Icarus struct { + firestore *firestoreClient +} + +// New constructs an Icarus source. projectID is the Firestore project ID +// (from the Firebase console) — passed explicitly rather than hard-coded so +// tests can point at an httptest server and so the real value lives in one +// place at the call site (Task 9), not buried in this package. +func New(httpClient *http.Client, projectID string) *Icarus { + return &Icarus{firestore: newFirestoreClient(projectID, httpClient)} +} + +var ( + _ source.ModSource = (*Icarus)(nil) + _ source.CapabilityReporter = (*Icarus)(nil) +) + +func (s *Icarus) ID() string { return "icarus" } +func (s *Icarus) Name() string { return "Icarus (Project Daedalus)" } + +// AuthURL/ExchangeToken: unsupported — Firestore reads here are public. +func (s *Icarus) AuthURL() string { return "" } +func (s *Icarus) ExchangeToken(ctx context.Context, code string) (*source.Token, error) { + return nil, fmt.Errorf("source %q: authentication: %w", s.ID(), source.ErrNotSupported) +} + +// GetDependencies: the modinfo.json v2 schema has no dependency field. +func (s *Icarus) GetDependencies(ctx context.Context, mod *domain.Mod) ([]domain.ModReference, error) { + return nil, fmt.Errorf("source %q: dependencies: %w", s.ID(), source.ErrNotSupported) +} + +func (s *Icarus) Capabilities() source.Capabilities { + return source.Capabilities{Search: true, Dependencies: false, Updates: true, Auth: false} +} + +func (s *Icarus) TypeLabel() string { return "built-in" } + +// Search fetches the whole mods collection and filters client-side — this +// catalog has no server-side query support to speak of, matching +// project_daedalus's own ModsController#find_mods approach. +func (s *Icarus) Search(ctx context.Context, query source.SearchQuery) (source.SearchResult, error) { + docs, err := s.firestore.listCollection(ctx, "mods") + if err != nil { + return source.SearchResult{}, fmt.Errorf("source %q: searching: %w", s.ID(), err) + } + + var mods []domain.Mod + q := strings.ToLower(query.Query) + for _, d := range docs { + m := mapDoc(d) + if q == "" || strings.Contains(strings.ToLower(m.Name), q) || + strings.Contains(strings.ToLower(m.Author), q) || + strings.Contains(strings.ToLower(m.Description), q) { + mods = append(mods, m) + } + } + + pageSize := query.PageSize + if pageSize <= 0 { + pageSize = 20 + } + page := query.Page + if page < 0 { + page = 0 + } + start := page * pageSize + if start > len(mods) { + start = len(mods) + } + end := start + pageSize + if end > len(mods) { + end = len(mods) + } + + return source.SearchResult{Mods: mods[start:end], TotalCount: len(mods), Page: page, PageSize: pageSize}, nil +} + +func (s *Icarus) GetMod(ctx context.Context, queryGameID, modID string) (*domain.Mod, error) { + doc, err := s.firestore.getDocument(ctx, "mods", modID) + if err != nil { + return nil, fmt.Errorf("source %q: fetching mod %s: %w", s.ID(), modID, err) + } + m := mapDoc(*doc) + return &m, nil +} + +// GetModFiles returns the mod's downloadable files (pak and/or exmodz — see +// modinfo.json v2 schema). A single file is marked primary, matching the +// existing custom.API convention. +func (s *Icarus) GetModFiles(ctx context.Context, mod *domain.Mod) ([]domain.DownloadableFile, error) { + doc, err := s.firestore.getDocument(ctx, "mods", mod.ID) + if err != nil { + return nil, fmt.Errorf("source %q: listing files for %s: %w", s.ID(), mod.ID, err) + } + filesField, _ := doc.Fields["files"].(map[string]any) + var out []domain.DownloadableFile + for _, kind := range []string{"pak", "exmodz"} { + rawURL, ok := filesField[kind].(string) + if !ok || rawURL == "" { + continue + } + out = append(out, domain.DownloadableFile{ + ID: kind, + Name: kind, + FileName: fileNameFromURL(rawURL, kind), + Category: strings.ToUpper(kind), + }) + } + if len(out) == 1 { + out[0].IsPrimary = true + } + return out, nil +} + +// GetDownloadURL re-fetches the mod document and returns the stored URL for +// fileID ("pak" or "exmodz") directly — no signing, matching a static-URL +// catalog rather than an OAuth-gated one. +func (s *Icarus) GetDownloadURL(ctx context.Context, mod *domain.Mod, fileID string) (string, error) { + doc, err := s.firestore.getDocument(ctx, "mods", mod.ID) + if err != nil { + return "", fmt.Errorf("source %q: download URL for %s: %w", s.ID(), fileID, err) + } + filesField, _ := doc.Fields["files"].(map[string]any) + rawURL, ok := filesField[fileID].(string) + if !ok || rawURL == "" { + return "", fmt.Errorf("source %q: file %s: no download URL", s.ID(), fileID) + } + return rawURL, nil +} + +// CheckUpdates compares each installed mod's stored version against the +// catalog's current version string (semantic-ish, per modinfo.json's +// "recommended" versioning note — not guaranteed strictly semver, so this +// uses domain.IsNewerVersion the same way custom.API does). +func (s *Icarus) CheckUpdates(ctx context.Context, installed []domain.InstalledMod) ([]domain.Update, error) { + var updates []domain.Update + var errs []error + for _, inst := range installed { + select { + case <-ctx.Done(): + return updates, ctx.Err() + default: + } + current, err := s.GetMod(ctx, gameID, inst.ID) + if err != nil { + errs = append(errs, err) + continue + } + if domain.IsNewerVersion(inst.Version, current.Version) { + updates = append(updates, domain.Update{InstalledMod: inst, NewVersion: current.Version}) + } + } + if len(errs) > 0 { + return updates, fmt.Errorf("source %q: %d update check(s) failed: %v", s.ID(), len(errs), errs[0]) + } + return updates, nil +} + +// mapDoc converts a decoded Firestore document into domain.Mod per the +// modinfo.json v2 schema (docs/plans/2026-07-29-icarus-exmod-pak-research.md). +func mapDoc(d firestoreDoc) domain.Mod { + str := func(key string) string { + s, _ := d.Fields[key].(string) + return s + } + return domain.Mod{ + ID: d.ID, + SourceID: "icarus", + GameID: gameID, + Name: str("name"), + Author: str("author"), + Version: str("version"), + Category: str("compatibility"), // Icarus week-build string, e.g. "w57" + Description: str("description"), + PictureURL: str("imageURL"), + SourceURL: str("readmeURL"), + } +} + +func fileNameFromURL(rawURL, fallbackExt string) string { + u, err := url.Parse(rawURL) + if err != nil || u.Path == "" { + return fallbackExt + } + base := path.Base(u.Path) + if base == "." || base == "/" { + return fallbackExt + } + return base +} diff --git a/internal/source/icarus/icarus_test.go b/internal/source/icarus/icarus_test.go new file mode 100644 index 0000000..7e2780d --- /dev/null +++ b/internal/source/icarus/icarus_test.go @@ -0,0 +1,85 @@ +package icarus + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/source" +) + +func modsListHandler(mods []map[string]any) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + docs := make([]map[string]any, len(mods)) + for i, m := range mods { + docs[i] = map[string]any{ + "name": "projects/p/databases/(default)/documents/mods/" + m["id"].(string), + "fields": m["fields"], + } + } + json.NewEncoder(w).Encode(map[string]any{"documents": docs}) //nolint:errcheck + } +} + +func TestIcarus_Search_FiltersClientSide(t *testing.T) { + srv := httptest.NewServer(modsListHandler([]map[string]any{ + {"id": "abc", "fields": map[string]any{ + "name": map[string]any{"stringValue": "Bear Mount"}, "author": map[string]any{"stringValue": "Jimk72"}, + "description": map[string]any{"stringValue": "Ride a bear"}, "version": map[string]any{"stringValue": "3.3"}, + "compatibility": map[string]any{"stringValue": "w57"}, + "files": map[string]any{"mapValue": map[string]any{"fields": map[string]any{"exmodz": map[string]any{"stringValue": "https://x/bear.exmodz"}}}}, + }}, + {"id": "def", "fields": map[string]any{ + "name": map[string]any{"stringValue": "Wolf Pack"}, "author": map[string]any{"stringValue": "Someone"}, + "description": map[string]any{"stringValue": "Tame wolves"}, "version": map[string]any{"stringValue": "1.0"}, + "files": map[string]any{"mapValue": map[string]any{"fields": map[string]any{"pak": map[string]any{"stringValue": "https://x/wolf.pak"}}}}, + }}, + })) + defer srv.Close() + + src := New(srv.Client(), "test-project") + src.firestore.baseURL = srv.URL + + result, err := src.Search(context.Background(), source.SearchQuery{Query: "bear"}) + if err != nil { + t.Fatalf("Search: %v", err) + } + if len(result.Mods) != 1 || result.Mods[0].Name != "Bear Mount" { + t.Fatalf("Search(%q) = %+v, want exactly Bear Mount", "bear", result.Mods) + } + if result.Mods[0].GameID != "icarus" { + t.Errorf("GameID = %q, want icarus", result.Mods[0].GameID) + } +} + +func TestIcarus_GetModFiles_ReturnsExmodzAndPak(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]any{ //nolint:errcheck + "name": "projects/p/databases/(default)/documents/mods/abc", + "fields": map[string]any{ + "name": map[string]any{"stringValue": "Bear Mount"}, + "files": map[string]any{"mapValue": map[string]any{"fields": map[string]any{ + "exmodz": map[string]any{"stringValue": "https://x/bear.exmodz"}, + }}}, + }, + }) + })) + defer srv.Close() + + src := New(srv.Client(), "test-project") + src.firestore.baseURL = srv.URL + + files, err := src.GetModFiles(context.Background(), &domain.Mod{ID: "abc", GameID: "icarus"}) + if err != nil { + t.Fatalf("GetModFiles: %v", err) + } + if len(files) != 1 || files[0].FileName != "bear.exmodz" { + t.Fatalf("files = %+v, want one bear.exmodz entry", files) + } + if !files[0].IsPrimary { + t.Error("single file should be marked primary") + } +} From 4d16cbf535ec8f9f34a9efb6d16e28bbe295493c Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Fri, 31 Jul 2026 20:02:01 -0400 Subject: [PATCH 08/96] feat: register Icarus as a built-in mod source (#136) Wires icarus.New into builtinSourceFactories using the real Project Daedalus Firestore project ID (projectdaedalus-fb09f), discovered via the daedalus-static-poc/AgentKush firebase config and live-verified read-only (538 mods, pagination, GetMod, GetModFiles). Documents the manual games.yaml entry until Steam auto-detection learns App ID 1149460. Co-Authored-By: Claude Sonnet 5 --- README.md | 12 +++++++++++- cmd/lmm/root.go | 8 ++++++++ cmd/lmm/root_test.go | 12 ++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1d72743..b7c4890 100644 --- a/README.md +++ b/README.md @@ -421,8 +421,18 @@ games: nexusmods: "starfield" link_method: copy # This game requires file copies instead of symlinks cache_path: /mnt/fast-ssd/starfield-mods # Store this game's mods on fast storage + + icarus: + name: "Icarus" + install_path: "/path/to/Steam/steamapps/common/Icarus" + mod_path: "/path/to/Steam/steamapps/common/Icarus/Icarus/Content/Paks/mods" + deploy_mode: compile # added in Task 13 + sources: + icarus: "icarus" ``` +Steam auto-detection (`lmm game detect`) does not yet know about Icarus (App ID `1149460`, confirmed during the research spike) — add this entry to `games.yaml` by hand for now; auto-detection is a separate, smaller follow-up not covered by this plan. + ### Deployment Methods Mods can be deployed using three methods: @@ -1085,7 +1095,7 @@ The mod cache location can be customized via `cache_path` in `config.yaml`. Sett - [x] Automatic dependency installation (opt out with `--no-deps`) - [x] Interactive TUI (Bubble Tea) - see the Terminal UI section above - [x] CurseForge integration -- [ ] Additional first-party built-in sources beyond NexusMods/CurseForge +- [x] Additional first-party built-in sources beyond NexusMods/CurseForge (Icarus) - [ ] Game auto-detection beyond Steam (Lutris, Heroic, Flatpak) - [ ] Backup and restore diff --git a/cmd/lmm/root.go b/cmd/lmm/root.go index 16c60a5..d3f9b33 100644 --- a/cmd/lmm/root.go +++ b/cmd/lmm/root.go @@ -15,6 +15,7 @@ import ( "github.com/DonovanMods/linux-mod-manager/internal/source" "github.com/DonovanMods/linux-mod-manager/internal/source/curseforge" "github.com/DonovanMods/linux-mod-manager/internal/source/custom" + "github.com/DonovanMods/linux-mod-manager/internal/source/icarus" "github.com/DonovanMods/linux-mod-manager/internal/source/nexusmods" "github.com/DonovanMods/linux-mod-manager/internal/storage/config" @@ -207,12 +208,19 @@ func initService() (*core.Service, error) { return svc, nil } +// icarusFirestoreProjectID is Project Daedalus's Firebase project ID, from +// the Firebase console. It is public information (Firestore reads are +// unauthenticated by design, per the research spike) — this constant is the +// one place it needs to be substituted with the real value. +const icarusFirestoreProjectID = "projectdaedalus-fb09f" + // builtinSourceFactories constructs each built-in source keyless — the // unified pipeline resolves and applies API keys post-construction via // registerSource's SetAPIKey seam, the same path custom sources use. var builtinSourceFactories = []func() source.ModSource{ func() source.ModSource { return nexusmods.New(nil, "") }, func() source.ModSource { return curseforge.New(nil, "") }, + func() source.ModSource { return icarus.New(nil, icarusFirestoreProjectID) }, } // registerSources registers all available mod sources with the service diff --git a/cmd/lmm/root_test.go b/cmd/lmm/root_test.go index f623bb1..6fd6797 100644 --- a/cmd/lmm/root_test.go +++ b/cmd/lmm/root_test.go @@ -192,6 +192,18 @@ directory: assert.Contains(t, warnBuf.String(), `warning: skipping source "nexusmods": id already in use`) } +func TestBuiltinSourceFactories_IncludesIcarus(t *testing.T) { + found := false + for _, factory := range builtinSourceFactories { + if factory().ID() == "icarus" { + found = true + } + } + if !found { + t.Error("builtinSourceFactories should include the icarus source") + } +} + func TestInitService_RegistersSources(t *testing.T) { // Use temp directories to avoid polluting real config configDir = t.TempDir() From dab5017b86f04e5eefb834da44005dbccda56c43 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Fri, 31 Jul 2026 20:07:26 -0400 Subject: [PATCH 09/96] feat: add .EXMOD parsing and row-patch application (#136) --- internal/source/icarus/exmod.go | 93 ++++++++++++++++++++++++++++ internal/source/icarus/exmod_test.go | 81 ++++++++++++++++++++++++ 2 files changed, 174 insertions(+) create mode 100644 internal/source/icarus/exmod.go create mode 100644 internal/source/icarus/exmod_test.go diff --git a/internal/source/icarus/exmod.go b/internal/source/icarus/exmod.go new file mode 100644 index 0000000..be3c335 --- /dev/null +++ b/internal/source/icarus/exmod.go @@ -0,0 +1,93 @@ +package icarus + +import ( + "encoding/json" + "fmt" +) + +// ExmodDiff is the parsed .EXMOD manifest — a diff against the base game's +// JSON data tables, not a binary/compiled-asset diff (confirmed against a +// real sample; see docs/plans/2026-07-29-icarus-exmod-pak-research.md). +type ExmodDiff struct { + Name string + Author string + Version string + Description string + Rows []ExmodRow +} + +// ExmodRow targets one base data-table file (e.g. "AI-D_AIGrowth.json"). +type ExmodRow struct { + CurrentFile string + FileItems []ExmodFileItem +} + +// ExmodFileItem overrides fields on the base row named Name. Fields holds +// every key from the source JSON except "Name" itself, generically — the +// real schema nests arbitrary game-data shapes here (see package doc +// comment), so this deliberately does not enumerate them. +type ExmodFileItem struct { + Name string + Fields map[string]any +} + +func ParseExmod(data []byte) (*ExmodDiff, error) { + var raw struct { + Name string `json:"name"` + Author string `json:"author"` + Version string `json:"version"` + Description string `json:"description"` + Rows []struct { + CurrentFile string `json:"CurrentFile"` + FileItems []map[string]any `json:"File_Items"` + } `json:"Rows"` + } + if err := json.Unmarshal(data, &raw); err != nil { + return nil, fmt.Errorf("icarus: parsing .EXMOD: %w", err) + } + + diff := &ExmodDiff{Name: raw.Name, Author: raw.Author, Version: raw.Version, Description: raw.Description} + for _, r := range raw.Rows { + row := ExmodRow{CurrentFile: r.CurrentFile} + for _, item := range r.FileItems { + name, _ := item["Name"].(string) + if name == "" { + return nil, fmt.Errorf("icarus: .EXMOD row in %s: File_Items entry missing Name", r.CurrentFile) + } + fields := make(map[string]any, len(item)-1) + for k, v := range item { + if k == "Name" { + continue + } + fields[k] = v + } + row.FileItems = append(row.FileItems, ExmodFileItem{Name: name, Fields: fields}) + } + diff.Rows = append(diff.Rows, row) + } + return diff, nil +} + +// ApplyRowPatch merges row's named-row field overrides into baseJSON (a base +// game data-table file keyed by row name, e.g. {"Mount_Bear": {...}, ...}) +// and returns the patched document. Fails loudly (no silent fallback, repo +// precedent #95) if a targeted row name doesn't exist in the base — that +// means either the base version is stale relative to the mod, or the exmod +// targets a file this function was called with by mistake. +func ApplyRowPatch(baseJSON []byte, row ExmodRow) ([]byte, error) { + var doc map[string]map[string]any + if err := json.Unmarshal(baseJSON, &doc); err != nil { + return nil, fmt.Errorf("icarus: parsing base data table %s: %w", row.CurrentFile, err) + } + for _, item := range row.FileItems { + target, ok := doc[item.Name] + if !ok { + return nil, fmt.Errorf("icarus: %s: row %q not found in base data table", row.CurrentFile, item.Name) + } + for k, v := range item.Fields { + target[k] = v + } + doc[item.Name] = target + } + return json.Marshal(doc) +} diff --git a/internal/source/icarus/exmod_test.go b/internal/source/icarus/exmod_test.go new file mode 100644 index 0000000..9b1d442 --- /dev/null +++ b/internal/source/icarus/exmod_test.go @@ -0,0 +1,81 @@ +package icarus + +import ( + "encoding/json" + "testing" +) + +const sampleExmod = `{ + "name": "Bear Mount", + "author": "Jimk72", + "version": "3.3", + "description": "Allows raising cubs", + "Rows": [ + { + "CurrentFile": "AI-D_AIGrowth.json", + "File_Items": [ + {"Name": "Mount_Bear", "BaseMovementSpeed": 235, "BaseSwimSpeed": 300} + ] + } + ] +}` + +func TestParseExmod(t *testing.T) { + diff, err := ParseExmod([]byte(sampleExmod)) + if err != nil { + t.Fatalf("ParseExmod: %v", err) + } + if diff.Name != "Bear Mount" || diff.Version != "3.3" { + t.Errorf("Name/Version = %q/%q, want Bear Mount/3.3", diff.Name, diff.Version) + } + if len(diff.Rows) != 1 || diff.Rows[0].CurrentFile != "AI-D_AIGrowth.json" { + t.Fatalf("Rows = %+v", diff.Rows) + } + if len(diff.Rows[0].FileItems) != 1 || diff.Rows[0].FileItems[0].Name != "Mount_Bear" { + t.Fatalf("FileItems = %+v", diff.Rows[0].FileItems) + } + if diff.Rows[0].FileItems[0].Fields["BaseMovementSpeed"] != float64(235) { + t.Errorf("BaseMovementSpeed = %v, want 235", diff.Rows[0].FileItems[0].Fields["BaseMovementSpeed"]) + } +} + +func TestApplyRowPatch_OverwritesNamedRowFieldsOnly(t *testing.T) { + base := []byte(`{ + "Mount_Bear": {"BaseMovementSpeed": 200, "BaseSwimSpeed": 150, "Untouched": "keep-me"}, + "Other_Row": {"BaseMovementSpeed": 999} + }`) + row := ExmodRow{ + CurrentFile: "AI-D_AIGrowth.json", + FileItems: []ExmodFileItem{ + {Name: "Mount_Bear", Fields: map[string]any{"BaseMovementSpeed": float64(235)}}, + }, + } + + got, err := ApplyRowPatch(base, row) + if err != nil { + t.Fatalf("ApplyRowPatch: %v", err) + } + + var result map[string]map[string]any + if err := json.Unmarshal(got, &result); err != nil { + t.Fatalf("unmarshaling result: %v", err) + } + if result["Mount_Bear"]["BaseMovementSpeed"] != float64(235) { + t.Errorf("BaseMovementSpeed not patched: %v", result["Mount_Bear"]["BaseMovementSpeed"]) + } + if result["Mount_Bear"]["Untouched"] != "keep-me" { + t.Errorf("unrelated field was clobbered: %v", result["Mount_Bear"]["Untouched"]) + } + if result["Other_Row"]["BaseMovementSpeed"] != float64(999) { + t.Errorf("unrelated row was modified: %v", result["Other_Row"]) + } +} + +func TestApplyRowPatch_UnknownRowName_Errors(t *testing.T) { + base := []byte(`{"Mount_Bear": {}}`) + row := ExmodRow{FileItems: []ExmodFileItem{{Name: "Does_Not_Exist", Fields: map[string]any{"X": 1}}}} + + if _, err := ApplyRowPatch(base, row); err == nil { + t.Error("expected error for unknown row name (no silent fallback), got nil") + } +} From 4e20ce6824c0689f79f621557d08c9c63bcac903 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Fri, 31 Jul 2026 20:12:29 -0400 Subject: [PATCH 10/96] feat: add .EXMODZ archive unpacking (#136) --- internal/source/icarus/exmodz.go | 74 +++++++++++++++++++++++++++ internal/source/icarus/exmodz_test.go | 71 +++++++++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 internal/source/icarus/exmodz.go create mode 100644 internal/source/icarus/exmodz_test.go diff --git a/internal/source/icarus/exmodz.go b/internal/source/icarus/exmodz.go new file mode 100644 index 0000000..d0c1a79 --- /dev/null +++ b/internal/source/icarus/exmodz.go @@ -0,0 +1,74 @@ +package icarus + +import ( + "archive/zip" + "bytes" + "fmt" + "io" + "strings" +) + +// ExmodzBundle is a parsed .EXMODZ: the diff manifest plus any pre-built +// asset files the mod author already compiled (placed as-is into the output +// pak — never recompiled by LMM). +type ExmodzBundle struct { + Diff *ExmodDiff + Assets map[string][]byte // zip-internal path -> raw content, manifest/readme/image excluded +} + +// ParseExmodz unpacks zipData (an in-memory .EXMODZ) into its manifest and +// bundled assets. The manifest lives at "Extracted Mods/.EXMOD" in +// every sample seen so far; this looks for any "*.EXMOD" file under an +// "Extracted Mods/" prefix rather than hard-coding the mod name, since that +// varies per mod. +func ParseExmodz(zipData []byte) (*ExmodzBundle, error) { + zr, err := zip.NewReader(bytes.NewReader(zipData), int64(len(zipData))) + if err != nil { + return nil, fmt.Errorf("icarus: opening .EXMODZ: %w", err) + } + + bundle := &ExmodzBundle{Assets: make(map[string][]byte)} + var manifestPath string + for _, f := range zr.File { + if strings.HasPrefix(f.Name, "Extracted Mods/") && strings.HasSuffix(f.Name, ".EXMOD") { + manifestPath = f.Name + data, err := readZipFile(f) + if err != nil { + return nil, fmt.Errorf("icarus: reading %s: %w", f.Name, err) + } + bundle.Diff, err = ParseExmod(data) + if err != nil { + return nil, err + } + continue + } + } + if manifestPath == "" { + return nil, fmt.Errorf("icarus: .EXMODZ has no Extracted Mods/*.EXMOD manifest") + } + + for _, f := range zr.File { + if f.Name == manifestPath || f.FileInfo().IsDir() { + continue + } + if !strings.HasSuffix(f.Name, ".uasset") && !strings.HasSuffix(f.Name, ".uexp") { + continue // skip readme/image/other non-asset files — never placed into the output pak + } + data, err := readZipFile(f) + if err != nil { + return nil, fmt.Errorf("icarus: reading asset %s: %w", f.Name, err) + } + bundle.Assets[f.Name] = data + } + + return bundle, nil +} + +func readZipFile(f *zip.File) ([]byte, error) { + rc, err := f.Open() + if err != nil { + return nil, err + } + defer rc.Close() //nolint:errcheck + return io.ReadAll(rc) +} diff --git a/internal/source/icarus/exmodz_test.go b/internal/source/icarus/exmodz_test.go new file mode 100644 index 0000000..bf73b82 --- /dev/null +++ b/internal/source/icarus/exmodz_test.go @@ -0,0 +1,71 @@ +package icarus + +import ( + "archive/zip" + "bytes" + "testing" +) + +// buildTestExmodz mirrors the real Bear_Mount.EXMODZ layout: a manifest +// under "Extracted Mods/.EXMOD" plus loose asset files at paths that +// mirror in-game mount structure. +func buildTestExmodz(t *testing.T) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + + manifest := `{"name":"Bear Mount","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":235}]}]}` + w, err := zw.Create("Extracted Mods/Bear_Mount.EXMOD") + if err != nil { + t.Fatal(err) + } + w.Write([]byte(manifest)) //nolint:errcheck + + assetW, err := zw.Create("Bear_Mount/ASS/ITM/SK_ITM_Saddle_Bear.uasset") + if err != nil { + t.Fatal(err) + } + assetW.Write([]byte("fake-uasset-bytes")) //nolint:errcheck + + if err := zw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func TestParseExmodz(t *testing.T) { + bundle, err := ParseExmodz(buildTestExmodz(t)) + if err != nil { + t.Fatalf("ParseExmodz: %v", err) + } + if bundle.Diff == nil || bundle.Diff.Name != "Bear Mount" { + t.Fatalf("Diff = %+v", bundle.Diff) + } + asset, ok := bundle.Assets["Bear_Mount/ASS/ITM/SK_ITM_Saddle_Bear.uasset"] + if !ok { + t.Fatalf("Assets missing expected key; got keys: %v", mapKeys(bundle.Assets)) + } + if string(asset) != "fake-uasset-bytes" { + t.Errorf("asset content = %q", asset) + } +} + +func mapKeys(m map[string][]byte) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} + +func TestParseExmodz_NoManifest_Errors(t *testing.T) { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, _ := zw.Create("readme.txt") + w.Write([]byte("no manifest here")) //nolint:errcheck + zw.Close() //nolint:errcheck + + if _, err := ParseExmodz(buf.Bytes()); err == nil { + t.Error("expected error when no .EXMOD manifest is present, got nil") + } +} From e25e2364a4c04ffec5b1583aa5133c195db82999 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Fri, 31 Jul 2026 20:17:39 -0400 Subject: [PATCH 11/96] feat: fetch Icarus base data tables from the per-week community dump (#136) --- internal/source/icarus/datadump.go | 291 ++++++++++++++++++++++++ internal/source/icarus/datadump_test.go | 254 +++++++++++++++++++++ 2 files changed, 545 insertions(+) create mode 100644 internal/source/icarus/datadump.go create mode 100644 internal/source/icarus/datadump_test.go diff --git a/internal/source/icarus/datadump.go b/internal/source/icarus/datadump.go new file mode 100644 index 0000000..02b7796 --- /dev/null +++ b/internal/source/icarus/datadump.go @@ -0,0 +1,291 @@ +package icarus + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "io/fs" + "net/http" + "os" + "path" + "path/filepath" + "sort" + "strings" + + "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" +) + +// defaultDumpTreeURL is the community per-week unpack of Icarus's data.pak: +// https://github.com/GODOFMINECRAFT4/IcarusData. The tree is committed as +// loose JSON at the repo root, one commit per game week, with the week +// recorded only in the commit message — there are no tags or releases. This +// URL is HEAD; a specific week is addressed by substituting its commit SHA. +const defaultDumpTreeURL = "https://codeload.github.com/GODOFMINECRAFT4/IcarusData/tar.gz/refs/heads/master" + +// maxDumpBytes caps the download. The real tree is ~36 MB; this leaves room to +// grow while refusing to stream an unbounded body into memory. +const maxDumpBytes = 256 << 20 + +// Build identifies the installed game, read from Icarus/Config/version.json. +// Note this carries no week number — nothing in the install does. Week +// agreement is established by content comparison, not by this value. +type Build struct { + Major, Minor, Patch int + Changelist int + DataChangelist int + FeatureLevel string +} + +func (b Build) String() string { + return fmt.Sprintf("%d.%d.%d.%d", b.Major, b.Minor, b.Patch, b.Changelist) +} + +// detectBuild reads /Icarus/Config/version.json. +func detectBuild(installRoot string) (Build, error) { + p := filepath.Join(installRoot, "Icarus", "Config", "version.json") + raw, err := os.ReadFile(p) + if err != nil { + return Build{}, fmt.Errorf("icarus: reading game version from %s: %w", p, err) + } + var doc struct { + Version struct { + Major, Minor, Patch int + Changelist int + FeatureLevel string + } + Data struct{ Changelist int } + } + if err := json.Unmarshal(raw, &doc); err != nil { + return Build{}, fmt.Errorf("icarus: parsing %s: %w", p, err) + } + return Build{ + Major: doc.Version.Major, Minor: doc.Version.Minor, Patch: doc.Version.Patch, + Changelist: doc.Version.Changelist, + DataChangelist: doc.Data.Changelist, + FeatureLevel: doc.Version.FeatureLevel, + }, nil +} + +// Dump is a fetched set of base data tables, keyed by mount-relative path +// (e.g. "Factions/D_Factions.json") with values already converted back to the +// game's CRLF line endings. +type Dump struct { + tables map[string][]byte +} + +// Table returns one table's shipped bytes. +func (d *Dump) Table(rel string) ([]byte, bool) { + b, ok := d.tables[rel] + return b, ok +} + +// DumpStore fetches and caches base-table dumps. +type DumpStore struct { + cacheDir string + httpClient *http.Client + treeURL string // overridable in tests +} + +func newDumpStore(cacheDir string, httpClient *http.Client) *DumpStore { + return &DumpStore{cacheDir: cacheDir, httpClient: httpClient, treeURL: defaultDumpTreeURL} +} + +// DumpForBuild loads the base data tables and returns them only if they match +// the installed game, proven by byte-comparing every table basePakPath stores +// uncompressed. A mismatch means the tables are for a different game week: +// that is a hard error naming the offending tables, never a silent +// best-effort. +// +// localDumpDir, when non-empty, is a user-supplied directory holding an +// unpacked data.pak JSON tree (QuickBMS output and the like); it replaces the +// network fetch entirely. Validation is the same either way — a local +// directory from the wrong week is rejected exactly like a stale hosted dump. +func (s *DumpStore) DumpForBuild(ctx context.Context, basePakPath, localDumpDir string) (*Dump, error) { + var ( + dump *Dump + err error + ) + if localDumpDir != "" { + dump, err = loadLocalDump(localDumpDir) + } else { + dump, err = s.fetchTree(ctx, s.treeURL) + } + if err != nil { + return nil, err + } + if err := validateDump(dump, basePakPath); err != nil { + if localDumpDir != "" { + return nil, fmt.Errorf("%w (tables were read from the configured data_dump_path %s)", err, localDumpDir) + } + return nil, err + } + return dump, nil +} + +// loadLocalDump reads an unpacked data.pak JSON tree from disk. The layout is +// the same one the hosted dump ships — table paths relative to the directory +// root, e.g. "Factions/D_Factions.json" — so a user can point this at QuickBMS +// output without rearranging anything. +func loadLocalDump(dir string) (*Dump, error) { + info, err := os.Stat(dir) + if err != nil { + return nil, fmt.Errorf("icarus: reading the configured data_dump_path %s: %w", dir, err) + } + if !info.IsDir() { + return nil, fmt.Errorf("icarus: the configured data_dump_path %s is not a directory", dir) + } + + dump := &Dump{tables: make(map[string][]byte)} + err = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(d.Name(), ".json") { + return nil + } + rel, err := filepath.Rel(dir, p) + if err != nil { + return err + } + body, err := os.ReadFile(p) + if err != nil { + return err + } + dump.tables[filepath.ToSlash(rel)] = toCRLF(body) + return nil + }) + if err != nil { + return nil, fmt.Errorf("icarus: scanning the configured data_dump_path %s: %w", dir, err) + } + if len(dump.tables) == 0 { + return nil, fmt.Errorf("icarus: the configured data_dump_path %s contains no JSON tables "+ + "(expected an unpacked data.pak tree, e.g. Factions/D_Factions.json)", dir) + } + return dump, nil +} + +// fetchTree downloads a dump tarball and ingests its JSON tables, restoring +// the CRLF line endings the game ships (the repo stores LF). +func (s *DumpStore) fetchTree(ctx context.Context, url string) (*Dump, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("icarus: building dump request: %w", err) + } + resp, err := s.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("icarus: fetching base-table dump: %w "+ + "(compiling Icarus mods requires network access — see the plan's Global Constraints)", err) + } + defer resp.Body.Close() //nolint:errcheck + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("icarus: fetching base-table dump from %s: HTTP %d", url, resp.StatusCode) + } + + zr, err := gzip.NewReader(io.LimitReader(resp.Body, maxDumpBytes)) + if err != nil { + return nil, fmt.Errorf("icarus: base-table dump is not valid gzip: %w", err) + } + defer zr.Close() //nolint:errcheck + + dump := &Dump{tables: make(map[string][]byte)} + tr := tar.NewReader(zr) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("icarus: reading base-table dump: %w", err) + } + if hdr.Typeflag != tar.TypeReg || !strings.HasSuffix(hdr.Name, ".json") { + continue + } + // Strip the archive's single top-level directory (e.g. + // "IcarusData-/") to get the mount-relative table path. + rel := hdr.Name + if i := strings.Index(rel, "/"); i >= 0 { + rel = rel[i+1:] + } + // The repo also carries a stale "data/" copy of the tree; the + // authoritative tables are the root-level ones. + if rel == "" || strings.HasPrefix(rel, "data/") { + continue + } + body, err := io.ReadAll(tr) + if err != nil { + return nil, fmt.Errorf("icarus: reading %s from base-table dump: %w", rel, err) + } + dump.tables[path.Clean(rel)] = toCRLF(body) + } + if len(dump.tables) == 0 { + return nil, fmt.Errorf("icarus: base-table dump from %s contained no JSON tables", url) + } + return dump, nil +} + +// toCRLF restores the game's line endings. The dump repo stores LF (committed +// with autocrlf); the shipped pak stores CRLF, and the two are otherwise +// byte-identical. Existing CRLFs are left alone so the conversion is +// idempotent. +func toCRLF(b []byte) []byte { + return []byte(strings.ReplaceAll(strings.ReplaceAll(string(b), "\r\n", "\n"), "\n", "\r\n")) +} + +// validateDump proves a dump belongs to the installed game. +// +// Only the tables data.pak stores *uncompressed* can be checked — the rest are +// Oodle-compressed and unreadable here, which is the whole reason the dump +// exists. That is enough: a dump built from a different week's data.pak +// disagrees on some of them, and in practice it disagrees loudly (the spike saw +// 3 differing stored tables and 6 missing tables across a 7-week gap). +func validateDump(dump *Dump, basePakPath string) error { + pak, err := unrealpak.Open(basePakPath) + if err != nil { + return fmt.Errorf("icarus: opening base pak %s for dump validation: %w", basePakPath, err) + } + defer pak.Close() //nolint:errcheck + + var missing, differing []string + checked := 0 + for _, f := range pak.Files() { + shipped, err := pak.ReadFile(f.Path) + if err != nil { + continue // Oodle-compressed: not readable here, and not our gate + } + checked++ + got, ok := dump.Table(f.Path) + if !ok { + missing = append(missing, f.Path) + continue + } + if !bytes.Equal(got, shipped) { + differing = append(differing, f.Path) + } + } + if checked == 0 { + return fmt.Errorf("icarus: %s exposed no uncompressed tables to validate the dump against", basePakPath) + } + if len(missing) == 0 && len(differing) == 0 { + return nil + } + sort.Strings(missing) + sort.Strings(differing) + return fmt.Errorf( + "icarus: the available base-table dump does not match the installed game "+ + "(%d/%d uncompressed tables disagree: %s). The dump is for a different game week. "+ + "Wait for the dump to be updated for your game version, or roll the game back to a "+ + "matching week; compiling against a mismatched week would silently corrupt mod data", + len(missing)+len(differing), checked, summarize(append(differing, missing...))) +} + +func summarize(paths []string) string { + const max = 3 + if len(paths) <= max { + return strings.Join(paths, ", ") + } + return fmt.Sprintf("%s and %d more", strings.Join(paths[:max], ", "), len(paths)-max) +} diff --git a/internal/source/icarus/datadump_test.go b/internal/source/icarus/datadump_test.go new file mode 100644 index 0000000..0b19b3f --- /dev/null +++ b/internal/source/icarus/datadump_test.go @@ -0,0 +1,254 @@ +package icarus + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" +) + +// writeTestBasePak builds a stored, unencrypted version-11 pak holding one +// entry per (mount-relative path, content) pair, via the Task 4 Writer. It +// stands in for Task 12's identically-named helper, which does not exist yet +// at this point in the plan's task order. +func writeTestBasePak(t *testing.T, files map[string][]byte) string { + t.Helper() + pakPath := filepath.Join(t.TempDir(), "data.pak") + w, err := unrealpak.Create(pakPath) + if err != nil { + t.Fatalf("creating test base pak: %v", err) + } + for rel, data := range files { + if err := w.AddFile(rel, data); err != nil { + t.Fatalf("AddFile(%q): %v", rel, err) + } + } + if err := w.Close(); err != nil { + t.Fatalf("closing test base pak: %v", err) + } + return pakPath +} + +// tarGz builds a dump-shaped tarball: a single top-level directory, then the +// table tree beneath it, LF-terminated exactly as the real repo stores it. +func tarGz(t *testing.T, root string, files map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + for name, body := range files { + hdr := &tar.Header{Name: root + "/" + name, Mode: 0o644, Size: int64(len(body))} + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(body)); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func TestDetectBuild_ReadsVersionJSON(t *testing.T) { + root := t.TempDir() + cfg := filepath.Join(root, "Icarus", "Config") + if err := os.MkdirAll(cfg, 0o755); err != nil { + t.Fatal(err) + } + const vjson = `{"Name":"Icarus","Version":{"Major":3,"Minor":0,"Patch":21,` + + `"Changelist":155335,"BuildType":"Shipping","FeatureLevel":"DangerousHorizons"},` + + `"Data":{"Changelist":155151}}` + if err := os.WriteFile(filepath.Join(cfg, "version.json"), []byte(vjson), 0o644); err != nil { + t.Fatal(err) + } + + b, err := detectBuild(root) + if err != nil { + t.Fatalf("detectBuild: %v", err) + } + if got := b.String(); got != "3.0.21.155335" { + t.Errorf("Build.String() = %q, want 3.0.21.155335", got) + } + if b.DataChangelist != 155151 { + t.Errorf("DataChangelist = %d, want 155151", b.DataChangelist) + } +} + +func TestDetectBuild_MissingVersionFile_Errors(t *testing.T) { + if _, err := detectBuild(t.TempDir()); err == nil { + t.Fatal("expected error when version.json is absent, got nil") + } +} + +// A dump whose stored tables match the local pak byte-for-byte (after CRLF +// restoration) is accepted, and its tables are exposed with shipped bytes. +func TestDumpStore_DumpForBuild_AcceptsMatchingDump(t *testing.T) { + const rel = "Factions/D_Factions.json" + shipped := []byte("{\r\n \"Rows\": []\r\n}") // CRLF, as the pak stores it + dumped := "{\n \"Rows\": []\n}" // LF, as the repo stores it + + pak := writeTestBasePak(t, map[string][]byte{rel: shipped}) // Task 12's helper + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(tarGz(t, "IcarusData-abc123", map[string]string{rel: dumped})) + })) + defer srv.Close() + + store := newDumpStore(t.TempDir(), srv.Client()) + store.treeURL = srv.URL // test seam + + dump, err := store.DumpForBuild(context.Background(), pak, "") + if err != nil { + t.Fatalf("DumpForBuild: %v", err) + } + got, ok := dump.Table(rel) + if !ok { + t.Fatalf("dump has no table %q", rel) + } + if !bytes.Equal(got, shipped) { + t.Errorf("table bytes = %q, want the shipped CRLF form %q", got, shipped) + } +} + +// The case that is live today: the newest dump is an older week than the +// install. Must fail loudly and name what disagreed. +func TestDumpStore_DumpForBuild_RejectsWrongWeek(t *testing.T) { + const rel = "Factions/D_Factions.json" + pak := writeTestBasePak(t, map[string][]byte{rel: []byte("{\r\n \"Rows\": [1]\r\n}")}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(tarGz(t, "IcarusData-old", map[string]string{rel: "{\n \"Rows\": []\n}"})) + })) + defer srv.Close() + + store := newDumpStore(t.TempDir(), srv.Client()) + store.treeURL = srv.URL + + _, err := store.DumpForBuild(context.Background(), pak, "") + if err == nil { + t.Fatal("expected an error for a dump that does not match the install, got nil") + } + if !strings.Contains(err.Error(), rel) { + t.Errorf("error %q should name the table that disagreed (%s)", err, rel) + } +} + +// writeLocalDump lays out an unpacked-data.pak-shaped directory on disk. +func writeLocalDump(t *testing.T, files map[string]string) string { + t.Helper() + dir := t.TempDir() + for rel, body := range files { + full := filepath.Join(dir, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + return dir +} + +// With a local dump directory configured, the network is never touched. +func TestDumpStore_DumpForBuild_LocalDirOverridesFetch(t *testing.T) { + const rel = "Factions/D_Factions.json" + shipped := []byte("{\r\n \"Rows\": []\r\n}") + pak := writeTestBasePak(t, map[string][]byte{rel: shipped}) + local := writeLocalDump(t, map[string]string{rel: "{\n \"Rows\": []\n}"}) + + fetched := false + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fetched = true + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + store := newDumpStore(t.TempDir(), srv.Client()) + store.treeURL = srv.URL + + dump, err := store.DumpForBuild(context.Background(), pak, local) + if err != nil { + t.Fatalf("DumpForBuild with a local dump dir: %v", err) + } + if fetched { + t.Error("the hosted dump was fetched even though a local dump dir was configured") + } + got, ok := dump.Table(rel) + if !ok || !bytes.Equal(got, shipped) { + t.Errorf("table bytes = %q (found=%v), want the shipped CRLF form %q", got, ok, shipped) + } +} + +// A local directory already storing CRLF must load unchanged — QuickBMS writes +// whatever the pak stored, so the conversion has to be idempotent. +func TestDumpStore_DumpForBuild_LocalDirAlreadyCRLF(t *testing.T) { + const rel = "Factions/D_Factions.json" + shipped := "{\r\n \"Rows\": []\r\n}" + pak := writeTestBasePak(t, map[string][]byte{rel: []byte(shipped)}) + local := writeLocalDump(t, map[string]string{rel: shipped}) + + store := newDumpStore(t.TempDir(), http.DefaultClient) + store.treeURL = "http://127.0.0.1:0/never-used" + + if _, err := store.DumpForBuild(context.Background(), pak, local); err != nil { + t.Fatalf("DumpForBuild with a CRLF local dump dir: %v", err) + } +} + +// A local dir from the wrong week is rejected exactly like a stale hosted +// dump, and the error points at the configured path. +func TestDumpStore_DumpForBuild_LocalDirWrongWeek_Rejected(t *testing.T) { + const rel = "Factions/D_Factions.json" + pak := writeTestBasePak(t, map[string][]byte{rel: []byte("{\r\n \"Rows\": [1]\r\n}")}) + local := writeLocalDump(t, map[string]string{rel: "{\n \"Rows\": []\n}"}) + + store := newDumpStore(t.TempDir(), http.DefaultClient) + store.treeURL = "http://127.0.0.1:0/never-used" + + _, err := store.DumpForBuild(context.Background(), pak, local) + if err == nil { + t.Fatal("expected an error for a local dump dir from a different week, got nil") + } + if !strings.Contains(err.Error(), rel) { + t.Errorf("error %q should name the disagreeing table (%s)", err, rel) + } + if !strings.Contains(err.Error(), local) { + t.Errorf("error %q should name the configured data_dump_path (%s)", err, local) + } +} + +func TestDumpStore_DumpForBuild_LocalDirEmpty_IsActionable(t *testing.T) { + pak := writeTestBasePak(t, map[string][]byte{"a/B.json": []byte("{}")}) + store := newDumpStore(t.TempDir(), http.DefaultClient) + + _, err := store.DumpForBuild(context.Background(), pak, t.TempDir()) + if err == nil { + t.Fatal("expected an error for a data_dump_path holding no JSON tables, got nil") + } +} + +func TestDumpStore_DumpForBuild_NetworkFailure_IsActionable(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + store := newDumpStore(t.TempDir(), srv.Client()) + store.treeURL = srv.URL + + pak := writeTestBasePak(t, map[string][]byte{"a/B.json": []byte("{}")}) + _, err := store.DumpForBuild(context.Background(), pak, "") + if err == nil { + t.Fatal("expected an error when the dump host fails, got nil") + } +} From 6b22ec67ffe27336a10a484340f30cfbea6df4de Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Fri, 31 Jul 2026 20:25:05 -0400 Subject: [PATCH 12/96] fix: distinguish unexpected pak read errors from Oodle skip in validateDump (#136) validateDump previously treated any pak.ReadFile error identically via a blanket continue, conflating the expected Oodle-compression skip with genuinely unexpected errors (corruption, truncation, I/O failure) and silently narrowing what the gate verified. Now only errors.Is(err, unrealpak.ErrUnsupportedFormat) is skipped; any other error returns a wrapped, actionable error naming the unreadable table. --- internal/source/icarus/datadump.go | 10 ++- internal/source/icarus/datadump_test.go | 86 +++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 1 deletion(-) diff --git a/internal/source/icarus/datadump.go b/internal/source/icarus/datadump.go index 02b7796..7fd8a1e 100644 --- a/internal/source/icarus/datadump.go +++ b/internal/source/icarus/datadump.go @@ -6,6 +6,7 @@ import ( "compress/gzip" "context" "encoding/json" + "errors" "fmt" "io" "io/fs" @@ -254,7 +255,14 @@ func validateDump(dump *Dump, basePakPath string) error { for _, f := range pak.Files() { shipped, err := pak.ReadFile(f.Path) if err != nil { - continue // Oodle-compressed: not readable here, and not our gate + if errors.Is(err, unrealpak.ErrUnsupportedFormat) { + continue // Oodle-compressed (or similar): not readable here, and not our gate + } + // Any other ReadFile failure — corruption, a truncated payload, an + // I/O error — is not an expected skip. Silently excluding it here + // would quietly narrow what this gate actually verified, exactly + // the "no silent fallbacks" failure this function exists to prevent. + return fmt.Errorf("icarus: validating base pak %s: reading %s: %w", basePakPath, f.Path, err) } checked++ got, ok := dump.Table(f.Path) diff --git a/internal/source/icarus/datadump_test.go b/internal/source/icarus/datadump_test.go index 0b19b3f..ba7b9a8 100644 --- a/internal/source/icarus/datadump_test.go +++ b/internal/source/icarus/datadump_test.go @@ -15,6 +15,48 @@ import ( "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" ) +// testStoredHeaderSize mirrors unrealpak's unexported storedHeaderSize (the +// 53-byte FPakEntry header preceding a stored file's payload). Duplicated +// here because these fixture-corruption helpers need to reach into pak bytes +// unrealpak itself does not expose, the same way reader_test.go pokes at raw +// offsets from inside package unrealpak. +const testStoredHeaderSize = 53 + +// corruptFirstEntryPayload flips the first byte of the alphabetically-first +// entry's payload (which the Writer always places at file offset +// testStoredHeaderSize, since entries are packed with no gap starting at 0). +// This breaks the entry's stored SHA1 without touching its header, producing +// a genuinely unexpected ReadFile error distinct from ErrUnsupportedFormat. +func corruptFirstEntryPayload(t *testing.T, pakPath string) { + t.Helper() + data, err := os.ReadFile(pakPath) + if err != nil { + t.Fatal(err) + } + data[testStoredHeaderSize] ^= 0xFF + if err := os.WriteFile(pakPath, data, 0o644); err != nil { + t.Fatal(err) + } +} + +// corruptFirstEntryCompressionMethod patches the alphabetically-first entry's +// on-disk header CompressionMethodIndex field (bytes 24:28 of the 53-byte +// header at file offset 0) to a nonzero value. This simulates the +// compression-refusal ReadFile takes for real Oodle-compressed entries +// without needing the Writer to emit actual compressed data, which it never +// does (it only ever produces stored, method-0 entries). +func corruptFirstEntryCompressionMethod(t *testing.T, pakPath string) { + t.Helper() + data, err := os.ReadFile(pakPath) + if err != nil { + t.Fatal(err) + } + data[24] = 1 + if err := os.WriteFile(pakPath, data, 0o644); err != nil { + t.Fatal(err) + } +} + // writeTestBasePak builds a stored, unencrypted version-11 pak holding one // entry per (mount-relative path, content) pair, via the Task 4 Writer. It // stands in for Task 12's identically-named helper, which does not exist yet @@ -252,3 +294,47 @@ func TestDumpStore_DumpForBuild_NetworkFailure_IsActionable(t *testing.T) { t.Fatal("expected an error when the dump host fails, got nil") } } + +// A base pak entry that fails to read for a reason OTHER than +// unrealpak.ErrUnsupportedFormat (corruption, a truncated payload, an I/O +// error) must fail validateDump loudly, not be silently folded into the +// "not our gate" skip that Oodle-compressed entries get. +func TestValidateDump_CorruptedStoredEntry_FailsLoudly(t *testing.T) { + const rel = "a/B.json" + pak := writeTestBasePak(t, map[string][]byte{rel: []byte("{\r\n}")}) + corruptFirstEntryPayload(t, pak) + + dump := &Dump{tables: map[string][]byte{rel: []byte("{\r\n}")}} + err := validateDump(dump, pak) + if err == nil { + t.Fatal("expected an error for a corrupted stored entry, got nil") + } + if strings.Contains(err.Error(), "different game week") { + t.Errorf("error %q should report the read failure, not the mismatch/wrong-week message "+ + "(a corrupted entry is not evidence of a stale dump)", err) + } + if !strings.Contains(err.Error(), rel) { + t.Errorf("error %q should name the unreadable table (%s)", err, rel) + } +} + +// An entry that refuses with unrealpak.ErrUnsupportedFormat (the real-world +// case: Oodle compression) must still be skipped, not treated as a +// validateDump failure — it is excluded from the check, not a reason to +// reject the dump. +func TestValidateDump_SkipsUnsupportedFormatEntry_NotAnError(t *testing.T) { + const compressedRel = "a/Apple.json" // sorts first -> lands at file offset 0 + const okRel = "z/Zebra.json" + okShipped := []byte("{\r\n \"Rows\": []\r\n}") + + pak := writeTestBasePak(t, map[string][]byte{ + compressedRel: []byte("{\r\n}"), + okRel: okShipped, + }) + corruptFirstEntryCompressionMethod(t, pak) + + dump := &Dump{tables: map[string][]byte{okRel: okShipped}} + if err := validateDump(dump, pak); err != nil { + t.Fatalf("validateDump with one ErrUnsupportedFormat entry and one matching entry: %v", err) + } +} From ce4d2f456515ef556a0e2be97a33fe02fdb5f32c Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Fri, 31 Jul 2026 20:39:23 -0400 Subject: [PATCH 13/96] feat: implement exmod compile orchestration (#136) Adds Compile(ctx, dumps, basePakPath, localDumpDir, exmodzPath, outputPakPath), wiring unrealpak, ParseExmodz/ApplyRowPatch (Task 10/11) and DumpForBuild (Task 12a) into the end-to-end .exmodz -> _P.pak orchestration. Two corrections to the brief, both empirically grounded against a real install + a real Bear_Mount.EXMODZ (coordinator-approved, see task-12-report.md plan delta): - resolveCurrentFile reconstructs the base pak mount path by converting every '-' in CurrentFile back to '/', not by suffix-matching a literal hyphenated filename -- the brief's literal-suffix algorithm matched 0/14 real rows against the real base pak. - Compile skips the real-world 'EndOfMod' sentinel/terminator row and fails loudly on any other row missing File_Items, rather than trying to resolve a sentinel as a data table. Also adds sanitizeAssetPath (coordinator-approved) to reject path-traversal and absolute bundled-asset entry names from a .EXMODZ before they reach the output pak's index. --- internal/source/icarus/compile.go | 181 ++++++++++++++++ internal/source/icarus/compile_test.go | 285 +++++++++++++++++++++++++ 2 files changed, 466 insertions(+) create mode 100644 internal/source/icarus/compile.go create mode 100644 internal/source/icarus/compile_test.go diff --git a/internal/source/icarus/compile.go b/internal/source/icarus/compile.go new file mode 100644 index 0000000..9825fff --- /dev/null +++ b/internal/source/icarus/compile.go @@ -0,0 +1,181 @@ +package icarus + +import ( + "context" + "fmt" + "os" + "path" + "strings" + + "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" +) + +// Compile reads exmodzPath's .EXMOD diff, applies it to the game's base data +// tables, bundles in any pre-built assets the .EXMODZ carries, and writes the +// result as a new pak at outputPakPath ready to deploy as-is. +// +// The base tables come from the community per-week dump (Task 12a), not from +// basePakPath: 258 of the 298 tables in a real data.pak are Oodle-compressed +// and cannot be read with the stdlib. basePakPath is still opened, for two +// things it alone can answer — which tables the installed game actually has +// (so a bare, hyphen-flattened CurrentFile resolves to a real mount path), +// and whether the dump +// is for the installed week (DumpForBuild byte-checks it against the tables +// the pak stores uncompressed). A dump that does not match fails the whole +// compile; see Task 12a. +// +// localDumpDir is the game's optional data_dump_path: when set, base tables +// are read from that directory instead of being fetched. It is validated +// identically, so a stale local directory fails just as loudly. +func Compile(ctx context.Context, dumps *DumpStore, basePakPath, localDumpDir, exmodzPath, outputPakPath string) error { + exmodzData, err := os.ReadFile(exmodzPath) + if err != nil { + return fmt.Errorf("icarus: reading %s: %w", exmodzPath, err) + } + bundle, err := ParseExmodz(exmodzData) + if err != nil { + return fmt.Errorf("icarus: %s: %w", exmodzPath, err) + } + + base, err := unrealpak.Open(basePakPath) + if err != nil { + return fmt.Errorf("icarus: opening base pak %s: %w", basePakPath, err) + } + defer base.Close() + + // Loaded and validated before anything is written, so a week mismatch or + // an offline machine fails before a half-built pak exists on disk. + dump, err := dumps.DumpForBuild(ctx, basePakPath, localDumpDir) + if err != nil { + return err + } + + out, err := unrealpak.Create(outputPakPath) + if err != nil { + return fmt.Errorf("icarus: creating %s: %w", outputPakPath, err) + } + + for _, row := range bundle.Diff.Rows { + if row.CurrentFile == endOfModSentinel { + // A known .EXMOD ecosystem terminator row: no File_Items, no + // corresponding data table. Not a row to resolve or patch. + continue + } + if len(row.FileItems) == 0 { + return fmt.Errorf("icarus: %s: row has no File_Items to apply (malformed .EXMOD manifest)", row.CurrentFile) + } + mountPath, err := resolveCurrentFile(base, row.CurrentFile) + if err != nil { + return err + } + baseData, ok := dump.Table(mountPath) + if !ok { + return fmt.Errorf("icarus: base data table %s is present in the installed game "+ + "but missing from the base-table dump", mountPath) + } + patched, err := ApplyRowPatch(baseData, row) + if err != nil { + return err + } + if err := out.AddFile(mountPath, patched); err != nil { + return fmt.Errorf("icarus: writing patched %s: %w", mountPath, err) + } + } + + for assetPath, data := range bundle.Assets { + safePath, err := sanitizeAssetPath(assetPath) + if err != nil { + return err + } + if err := out.AddFile(safePath, data); err != nil { + return fmt.Errorf("icarus: writing bundled asset %s: %w", safePath, err) + } + } + + if err := out.Close(); err != nil { + return fmt.Errorf("icarus: finalizing %s: %w", outputPakPath, err) + } + return nil +} + +// endOfModSentinel is a known .EXMOD ecosystem terminator row: real-world +// manifests end their Rows array with {"CurrentFile":"EndOfMod"} and no +// File_Items key at all. It targets no data table and carries no patch, so +// Compile skips it rather than trying (and failing) to resolve it. +const endOfModSentinel = "EndOfMod" + +// resolveCurrentFile finds the base-pak file a row's bare CurrentFile refers +// to. The .EXMOD schema flattens the mount-relative directory path into +// CurrentFile by replacing every "/" with "-" (e.g. the real base pak path +// "Audio/MusicConditions/D_MusicLocationConditions.json" is recorded as +// "Audio-MusicConditions-D_MusicLocationConditions.json"); reversing that +// substitution reconstructs the mount path exactly. This was verified +// against a real install and a real .EXMODZ: none of Icarus's 298 real base +// table paths contain a literal hyphen, so the reverse mapping is +// unambiguous. Fails loudly on zero or multiple matches — see this task's +// header note; guessing which one is correct is exactly the kind of silent +// fallback repo precedent #95 forbids. +func resolveCurrentFile(base *unrealpak.Reader, currentFile string) (string, error) { + files := base.Files() + paths := make([]string, len(files)) + for i, f := range files { + paths[i] = f.Path + } + return matchMountPath(paths, currentFile) +} + +// matchMountPath resolves currentFile against paths, isolated from +// *unrealpak.Reader so the zero/ambiguous-match error paths can be tested +// directly without needing a base pak with (unreachable in valid data) +// duplicate mount entries. +func matchMountPath(paths []string, currentFile string) (string, error) { + candidate := strings.ReplaceAll(currentFile, "-", "/") + var matches []string + for _, p := range paths { + if p == candidate { + matches = append(matches, p) + } + } + switch len(matches) { + case 1: + return matches[0], nil + case 0: + return "", fmt.Errorf("icarus: %s: no matching file in base pak "+ + "(expected mount path %s, from CurrentFile with '-' converted to '/')", currentFile, candidate) + default: + return "", fmt.Errorf("icarus: %s: ambiguous, matches %v", currentFile, matches) + } +} + +// sanitizeAssetPath validates a bundled asset's mount path before it is +// written into the output pak. .EXMODZ archives are third-party zip files, +// and ParseExmodz (Task 11) carries each entry's raw zip name through +// unchanged as the Assets map key. Without this gate, a crafted entry name +// (a "../" parent traversal, an absolute path, or a Windows drive path) could +// escape the mod's own namespace once the pak is deployed or unpacked +// elsewhere — the pak equivalent of a zip-slip. Rejecting it here, before +// AddFile, keeps that malformed-archive class of input a loud compile +// failure rather than a written-then-discovered problem. +func sanitizeAssetPath(rawZipName string) (string, error) { + normalized := strings.ReplaceAll(rawZipName, `\`, "/") + if strings.Contains(normalized, "\x00") { + return "", fmt.Errorf("icarus: bundled asset %q: contains a NUL byte", rawZipName) + } + if strings.HasPrefix(normalized, "/") || isWindowsDriveAbsolute(normalized) { + return "", fmt.Errorf("icarus: bundled asset %q: absolute paths are not allowed", rawZipName) + } + cleaned := path.Clean(normalized) + if cleaned == "." || cleaned == ".." || strings.HasPrefix(cleaned, "../") { + return "", fmt.Errorf("icarus: bundled asset %q: escapes the mod's own path", rawZipName) + } + return cleaned, nil +} + +// isWindowsDriveAbsolute reports whether p starts with a Windows drive letter +// (e.g. "C:/evil"). Checked on the slash-normalized form, since a zip entry +// written by a Windows tool may carry "C:\evil" — backslashes normalize to +// forward slashes before this check runs. +func isWindowsDriveAbsolute(p string) bool { + return len(p) >= 2 && p[1] == ':' && + ((p[0] >= 'A' && p[0] <= 'Z') || (p[0] >= 'a' && p[0] <= 'z')) +} diff --git a/internal/source/icarus/compile_test.go b/internal/source/icarus/compile_test.go new file mode 100644 index 0000000..8a8f588 --- /dev/null +++ b/internal/source/icarus/compile_test.go @@ -0,0 +1,285 @@ +package icarus + +import ( + "archive/zip" + "bytes" + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" +) + +// testDumpStore serves a dump containing exactly files, so validateDump agrees +// it matches the base pak built from the same map. Reuses tarGz from +// datadump_test.go (same package). +func testDumpStore(t *testing.T, files map[string][]byte) *DumpStore { + t.Helper() + entries := make(map[string]string, len(files)) + for name, data := range files { + entries[name] = string(data) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(tarGz(t, "IcarusData-test", entries)) + })) + t.Cleanup(srv.Close) + store := newDumpStore(t.TempDir(), srv.Client()) + store.treeURL = srv.URL + return store +} + +func writeTestExmodzFile(t *testing.T, manifestJSON string, assets map[string][]byte) string { + t.Helper() + path := filepath.Join(t.TempDir(), "mod.exmodz") + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, _ := zw.Create("Extracted Mods/Test.EXMOD") + w.Write([]byte(manifestJSON)) //nolint:errcheck + for name, data := range assets { + aw, _ := zw.Create(name) + aw.Write(data) //nolint:errcheck + } + zw.Close() //nolint:errcheck + if err := os.WriteFile(path, buf.Bytes(), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +// Base table paths and CurrentFile values below mirror the real shape found +// against a live install + a real Bear_Mount.EXMODZ during Step 5b +// verification: CurrentFile flattens the mount-relative directory path with +// "-" in place of "/" (e.g. "AI-D_AIGrowth.json" for base pak path +// "AI/D_AIGrowth.json"), not a bare filename living at a hyphenated leaf as +// the original brief's fixtures assumed. See task-12-report.md "plan delta". +func TestCompile_AppliesDiffAndBundlesAssets(t *testing.T) { + baseTables := map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":200}}`), + } + basePak := writeTestBasePak(t, baseTables) + dumps := testDumpStore(t, baseTables) + manifest := `{"name":"Bear Mount","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":235}]}]}` + exmodzPath := writeTestExmodzFile(t, manifest, map[string][]byte{ + "Bear_Mount/ASS/ITM/SK_ITM_Saddle_Bear.uasset": []byte("fake-asset"), + }) + outputPath := filepath.Join(t.TempDir(), "Bear_Mount_P.pak") + + if err := Compile(context.Background(), dumps, basePak, "", exmodzPath, outputPath); err != nil { + t.Fatalf("Compile: %v", err) + } + + r, err := unrealpak.Open(outputPath) + if err != nil { + t.Fatalf("opening compiled output: %v", err) + } + defer r.Close() + + patched, err := r.ReadFile("AI/D_AIGrowth.json") + if err != nil { + t.Fatalf("ReadFile patched data table: %v", err) + } + if !bytes.Contains(patched, []byte(`"BaseMovementSpeed":235`)) { + t.Errorf("patched data table = %s, want BaseMovementSpeed 235", patched) + } + + asset, err := r.ReadFile("Bear_Mount/ASS/ITM/SK_ITM_Saddle_Bear.uasset") + if err != nil { + t.Fatalf("ReadFile bundled asset: %v", err) + } + if string(asset) != "fake-asset" { + t.Errorf("bundled asset content = %q", asset) + } +} + +// The real .EXMOD ecosystem terminates Rows with {"CurrentFile":"EndOfMod"} +// and no File_Items key — Compile must skip it, not try to resolve it as a +// data table (it has none). +func TestCompile_SkipsEndOfModSentinelRow(t *testing.T) { + baseTables := map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":200}}`), + } + basePak := writeTestBasePak(t, baseTables) + dumps := testDumpStore(t, baseTables) + manifest := `{"name":"X","Rows":[` + + `{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":235}]},` + + `{"CurrentFile":"EndOfMod"}]}` + exmodzPath := writeTestExmodzFile(t, manifest, nil) + outputPath := filepath.Join(t.TempDir(), "out.pak") + + if err := Compile(context.Background(), dumps, basePak, "", exmodzPath, outputPath); err != nil { + t.Fatalf("Compile: %v", err) + } + + r, err := unrealpak.Open(outputPath) + if err != nil { + t.Fatalf("opening compiled output: %v", err) + } + defer r.Close() + patched, err := r.ReadFile("AI/D_AIGrowth.json") + if err != nil { + t.Fatalf("ReadFile patched data table: %v", err) + } + if !bytes.Contains(patched, []byte(`"BaseMovementSpeed":235`)) { + t.Errorf("patched data table = %s, want BaseMovementSpeed 235", patched) + } +} + +// A real (non-sentinel) row with no File_Items is a malformed manifest, not +// something to silently skip — only the EndOfMod sentinel gets that pass. +func TestCompile_RowWithoutFileItems_Errors(t *testing.T) { + baseTables := map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":200}}`), + } + basePak := writeTestBasePak(t, baseTables) + dumps := testDumpStore(t, baseTables) + manifest := `{"name":"X","Rows":[{"CurrentFile":"AI-D_AIGrowth.json"}]}` + exmodzPath := writeTestExmodzFile(t, manifest, nil) + outputPath := filepath.Join(t.TempDir(), "out.pak") + + err := Compile(context.Background(), dumps, basePak, "", exmodzPath, outputPath) + if err == nil { + t.Fatal("expected an error for a non-sentinel row with no File_Items, got nil") + } + if !strings.Contains(err.Error(), "AI-D_AIGrowth.json") { + t.Errorf("error %q should name the offending row", err) + } +} + +// A stale dump must stop the compile before any output pak is written — this +// is the live case today, where the newest dump lags the installed game. +func TestCompile_DumpWeekMismatch_FailsBeforeWriting(t *testing.T) { + basePak := writeTestBasePak(t, map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":200}}`), + }) + dumps := testDumpStore(t, map[string][]byte{ // different week's content + "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":150}}`), + }) + manifest := `{"name":"X","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":235}]}]}` + exmodzPath := writeTestExmodzFile(t, manifest, nil) + outputPath := filepath.Join(t.TempDir(), "out.pak") + + err := Compile(context.Background(), dumps, basePak, "", exmodzPath, outputPath) + if err == nil { + t.Fatal("expected an error when the dump is for a different game week, got nil") + } + if _, statErr := os.Stat(outputPath); statErr == nil { + t.Error("no output pak should exist after a week-mismatch failure") + } +} + +// A malicious .EXMODZ whose bundled asset entry escapes the mod's own path +// must fail loudly rather than write outside the pak's intended namespace — +// see task-12-report.md "plan delta" for the exact semantics agreed with the +// coordinator. +func TestCompile_UnsafeAssetPath_Errors(t *testing.T) { + baseTables := map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":200}}`), + } + basePak := writeTestBasePak(t, baseTables) + dumps := testDumpStore(t, baseTables) + manifest := `{"name":"X","Rows":[]}` + exmodzPath := writeTestExmodzFile(t, manifest, map[string][]byte{ + "../evil.uasset": []byte("payload"), + }) + outputPath := filepath.Join(t.TempDir(), "out.pak") + + err := Compile(context.Background(), dumps, basePak, "", exmodzPath, outputPath) + if err == nil { + t.Fatal("expected an error for an asset path escaping the mod's own namespace, got nil") + } + if !strings.Contains(err.Error(), "../evil.uasset") { + t.Errorf("error %q should name the offending asset path", err) + } +} + +func TestMatchMountPath(t *testing.T) { + paths := []string{ + "AI/D_AIGrowth.json", + "Audio/MusicConditions/D_MusicLocationConditions.json", + "D_Factions.json", + } + + t.Run("single-level directory", func(t *testing.T) { + got, err := matchMountPath(paths, "AI-D_AIGrowth.json") + if err != nil { + t.Fatalf("matchMountPath: %v", err) + } + if got != "AI/D_AIGrowth.json" { + t.Errorf("matchMountPath = %q, want AI/D_AIGrowth.json", got) + } + }) + + t.Run("multi-level directory", func(t *testing.T) { + got, err := matchMountPath(paths, "Audio-MusicConditions-D_MusicLocationConditions.json") + if err != nil { + t.Fatalf("matchMountPath: %v", err) + } + if got != "Audio/MusicConditions/D_MusicLocationConditions.json" { + t.Errorf("matchMountPath = %q, want Audio/MusicConditions/D_MusicLocationConditions.json", got) + } + }) + + t.Run("root-level file, no hyphen to convert", func(t *testing.T) { + got, err := matchMountPath(paths, "D_Factions.json") + if err != nil { + t.Fatalf("matchMountPath: %v", err) + } + if got != "D_Factions.json" { + t.Errorf("matchMountPath = %q, want D_Factions.json", got) + } + }) + + t.Run("no match is a loud, actionable error", func(t *testing.T) { + _, err := matchMountPath(paths, "AI-D_Nonexistent.json") + if err == nil { + t.Fatal("expected an error for a CurrentFile with no matching base pak file, got nil") + } + if !strings.Contains(err.Error(), "AI-D_Nonexistent.json") || !strings.Contains(err.Error(), "AI/D_Nonexistent.json") { + t.Errorf("error %q should name both the CurrentFile and the expected mount path", err) + } + }) + + t.Run("ambiguous match is a loud error", func(t *testing.T) { + dup := []string{"AI/D_AIGrowth.json", "AI/D_AIGrowth.json"} + _, err := matchMountPath(dup, "AI-D_AIGrowth.json") + if err == nil { + t.Fatal("expected an error for an ambiguous match, got nil") + } + }) +} + +func TestSanitizeAssetPath(t *testing.T) { + tests := []struct { + name string + raw string + want string + wantErr bool + }{ + {name: "parent traversal", raw: "../evil.json", wantErr: true}, + {name: "absolute unix path", raw: "/evil", wantErr: true}, + {name: "windows drive absolute", raw: `C:\evil`, wantErr: true}, + {name: "backslash-normalized nested path", raw: `Good\Nested\file.uasset`, want: "Good/Nested/file.uasset"}, + {name: "benign nested path", raw: "Bear_Mount/ASS/ITM/SK_ITM_Saddle_Bear.uasset", want: "Bear_Mount/ASS/ITM/SK_ITM_Saddle_Bear.uasset"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := sanitizeAssetPath(tt.raw) + if tt.wantErr { + if err == nil { + t.Fatalf("sanitizeAssetPath(%q) = %q, nil; want error", tt.raw, got) + } + return + } + if err != nil { + t.Fatalf("sanitizeAssetPath(%q): %v", tt.raw, err) + } + if got != tt.want { + t.Errorf("sanitizeAssetPath(%q) = %q, want %q", tt.raw, got, tt.want) + } + }) + } +} From 91198e98c5eb253514c136d91dd49c23d48aabfd Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Fri, 31 Jul 2026 20:50:29 -0400 Subject: [PATCH 14/96] fix: remove partial output pak on mid-compile failure (#136) Compile now uses a named return + deferred cleanup: once unrealpak.Create(outputPakPath) succeeds, any later error (unresolvable row, missing dump table, patch failure, unsafe asset path, AddFile/Close failure) removes the partial output file before returning, instead of leaving a stray incomplete _P.pak on disk. A removal failure is joined into the returned error rather than masking it; the success path is untouched. unrealpak.Writer has no abort-without-finalizing method, so the underlying file descriptor is only reclaimed on GC in this case -- os.Remove still eliminates the on-disk deploy hazard the review flagged. Adds TestCompile_MidCompileFailure_LeavesNoOutputFile and extends TestCompile_UnsafeAssetPath_Errors to assert no file exists at outputPakPath after a mid-compile failure. Fix round 1 for the Task 12 review's one Important finding (task-12-review.md). --- internal/source/icarus/compile.go | 17 +++++++++++++- internal/source/icarus/compile_test.go | 32 ++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/internal/source/icarus/compile.go b/internal/source/icarus/compile.go index 9825fff..384cb25 100644 --- a/internal/source/icarus/compile.go +++ b/internal/source/icarus/compile.go @@ -27,7 +27,7 @@ import ( // localDumpDir is the game's optional data_dump_path: when set, base tables // are read from that directory instead of being fetched. It is validated // identically, so a stale local directory fails just as loudly. -func Compile(ctx context.Context, dumps *DumpStore, basePakPath, localDumpDir, exmodzPath, outputPakPath string) error { +func Compile(ctx context.Context, dumps *DumpStore, basePakPath, localDumpDir, exmodzPath, outputPakPath string) (err error) { exmodzData, err := os.ReadFile(exmodzPath) if err != nil { return fmt.Errorf("icarus: reading %s: %w", exmodzPath, err) @@ -54,6 +54,21 @@ func Compile(ctx context.Context, dumps *DumpStore, basePakPath, localDumpDir, e if err != nil { return fmt.Errorf("icarus: creating %s: %w", outputPakPath, err) } + // unrealpak.Create opens the file eagerly, so any error from here on + // leaves a partial/incomplete pak at outputPakPath unless removed — a + // hazard, since it could be picked up and deployed. unrealpak.Writer has + // no way to abort without finalizing (Close always serializes and writes + // whatever was buffered), so removing the file is the only way to keep + // the fail-loud-and-clean contract on this path; the success path + // (err == nil here) is untouched. + defer func() { + if err == nil { + return + } + if rmErr := os.Remove(outputPakPath); rmErr != nil && !os.IsNotExist(rmErr) { + err = fmt.Errorf("%w (additionally, removing partial output %s failed: %v)", err, outputPakPath, rmErr) + } + }() for _, row := range bundle.Diff.Rows { if row.CurrentFile == endOfModSentinel { diff --git a/internal/source/icarus/compile_test.go b/internal/source/icarus/compile_test.go index 8a8f588..6614c77 100644 --- a/internal/source/icarus/compile_test.go +++ b/internal/source/icarus/compile_test.go @@ -194,6 +194,38 @@ func TestCompile_UnsafeAssetPath_Errors(t *testing.T) { if !strings.Contains(err.Error(), "../evil.uasset") { t.Errorf("error %q should name the offending asset path", err) } + if _, statErr := os.Stat(outputPath); statErr == nil { + t.Error("no partial output pak should exist after an unsafe-asset-path failure") + } +} + +// A failure that happens after unrealpak.Create(outputPakPath) has already +// created the file on disk (here: an unresolvable row, mid row-loop) must +// not leave a partial/incomplete pak behind — a stray partial _P.pak is a +// hazard (it could be picked up and deployed) and contradicts the +// fail-loud-and-clean philosophy. See task-12-report.md "plan delta" (fix +// round 1). +func TestCompile_MidCompileFailure_LeavesNoOutputFile(t *testing.T) { + baseTables := map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":200}}`), + } + basePak := writeTestBasePak(t, baseTables) + dumps := testDumpStore(t, baseTables) + // CurrentFile has no matching base-pak file: resolveCurrentFile fails + // inside the row loop, after out has already been created. + manifest := `{"name":"X","Rows":[{"CurrentFile":"AI-D_Nonexistent.json","File_Items":[{"Name":"Mount_Bear","X":1}]}]}` + exmodzPath := writeTestExmodzFile(t, manifest, nil) + outputPath := filepath.Join(t.TempDir(), "out.pak") + + err := Compile(context.Background(), dumps, basePak, "", exmodzPath, outputPath) + if err == nil { + t.Fatal("expected an error for an unresolvable row, got nil") + } + if _, statErr := os.Stat(outputPath); statErr == nil { + t.Error("no partial output pak should exist after a mid-compile failure") + } else if !os.IsNotExist(statErr) { + t.Errorf("unexpected error stat-ing output path: %v", statErr) + } } func TestMatchMountPath(t *testing.T) { From 9ec6978d0526c36e4ac9d10f66846f7d433f8b64 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Fri, 31 Jul 2026 21:12:22 -0400 Subject: [PATCH 15/96] feat: wire exmod compile step into cache-population pipeline (#136) Adds domain.DeployCompile, source.Compiler, and Icarus.Compile, and wires a DeployCompile branch into Service.DownloadModToCache: after download, a source implementing Compiler transforms the file (Icarus's .exmodz -> _P.pak) before it's committed to cache, so everything downstream treats it like a DeployCopy file. Adds the per-game data_dump_path config (games.yaml -> GameConfig -> domain.Game.BaseDataPath), and closes the Task 9 review gap where ParseDeployMode didn't yet recognize "compile". Icarus.Compile needs a real dumps cache directory in production, but New(httpClient, projectID) was frozen at those two params by an earlier task with a dependent call site. Wires it instead via an optional SetDataDir(string) setter, mirroring the existing SetAPIKey optional-setter pattern in cmd/lmm/root.go's registerSource; Compile fails loudly rather than panicking if SetDataDir was never called. Co-Authored-By: Claude Sonnet 5 --- README.md | 4 +- cmd/lmm/root.go | 26 +++-- cmd/lmm/root_test.go | 42 ++++++- docs/configuration.md | 22 ++-- internal/core/service.go | 56 ++++++++++ internal/core/service_icarus_compile_test.go | 110 +++++++++++++++++++ internal/domain/game.go | 8 ++ internal/domain/game_test.go | 2 + internal/source/icarus/icarus.go | 27 +++++ internal/source/icarus/icarus_test.go | 28 +++++ internal/source/source.go | 16 +++ internal/storage/config/games.go | 29 ++--- internal/storage/config/games_test.go | 24 ++++ 13 files changed, 355 insertions(+), 39 deletions(-) create mode 100644 internal/core/service_icarus_compile_test.go create mode 100644 internal/storage/config/games_test.go diff --git a/README.md b/README.md index b7c4890..f743786 100644 --- a/README.md +++ b/README.md @@ -426,7 +426,9 @@ games: name: "Icarus" install_path: "/path/to/Steam/steamapps/common/Icarus" mod_path: "/path/to/Steam/steamapps/common/Icarus/Icarus/Content/Paks/mods" - deploy_mode: compile # added in Task 13 + deploy_mode: compile + # data_dump_path: ~/icarus-data-dump # Optional: compile from your own + # unpacked data.pak JSON tree instead of the hosted community dump sources: icarus: "icarus" ``` diff --git a/cmd/lmm/root.go b/cmd/lmm/root.go index d3f9b33..6d831f4 100644 --- a/cmd/lmm/root.go +++ b/cmd/lmm/root.go @@ -203,7 +203,7 @@ func initService() (*core.Service, error) { } // Register mod sources - registerSources(svc, cfg.ConfigDir) + registerSources(svc, cfg.ConfigDir, cfg.DataDir) return svc, nil } @@ -226,21 +226,26 @@ var builtinSourceFactories = []func() source.ModSource{ // registerSources registers all available mod sources with the service // through one ordered pipeline: built-ins first (so the collision rule's // "first wins" preserves their identity against a same-id custom -// definition), then user-defined sources from /sources/. -func registerSources(svc *core.Service, cfgDir string) { +// definition), then user-defined sources from /sources/. dataDir +// is threaded through to registerSource for DeployCompile sources (#136 +// Task 13) that need it wired via the SetDataDir optional setter. +func registerSources(svc *core.Service, cfgDir, dataDir string) { for _, factory := range builtinSourceFactories { - registerSource(svc, factory()) + registerSource(svc, factory(), dataDir) } - registerCustomSources(svc, cfgDir) + registerCustomSources(svc, cfgDir, dataDir) } // registerSource runs src through the shared registration steps used for // both built-in and custom sources: collision check (first registration // wins, warning on customSourceWarnWriter) → API-key resolution (env var via // envKeyFor, falling back to the stored DB token) → SetAPIKey when the -// source accepts one → RegisterSource. -func registerSource(svc *core.Service, src source.ModSource) { +// source accepts one → SetDataDir when the source accepts one (Icarus's +// Compile needs a cache directory for the base-table dump store, #136 Task +// 13 — New itself can't take it since Task 8/9 froze its 2-arg signature) → +// RegisterSource. +func registerSource(svc *core.Service, src source.ModSource, dataDir string) { id := src.ID() // Custom sources are constructed (custom.New) by the caller before this // runs; a definition that both collides with an existing ID AND fails to @@ -259,6 +264,9 @@ func registerSource(svc *core.Service, src source.ModSource) { setter.SetAPIKey(key) } } + if setter, ok := src.(interface{ SetDataDir(string) }); ok { + setter.SetDataDir(dataDir) + } svc.RegisterSource(src) } @@ -295,7 +303,7 @@ func customSourceWarnWriter() io.Writer { // registerCustomSources loads user-defined source definitions and registers // the valid ones. Broken definitions warn (via customSourceWarnWriter, normally // os.Stderr) and are skipped — a bad file must never prevent lmm from starting. -func registerCustomSources(svc *core.Service, cfgDir string) { +func registerCustomSources(svc *core.Service, cfgDir, dataDir string) { defs, loadErrs, err := config.LoadSourceDefinitions(cfgDir) if err != nil { fmt.Fprintf(customSourceWarnWriter(), "warning: loading custom sources: %v\n", err) @@ -310,7 +318,7 @@ func registerCustomSources(svc *core.Service, cfgDir string) { fmt.Fprintf(customSourceWarnWriter(), "warning: skipping source %q: %v\n", def.ID, err) continue } - registerSource(svc, src) + registerSource(svc, src, dataDir) } } diff --git a/cmd/lmm/root_test.go b/cmd/lmm/root_test.go index 6fd6797..62e45f7 100644 --- a/cmd/lmm/root_test.go +++ b/cmd/lmm/root_test.go @@ -48,7 +48,7 @@ func TestRegisterSources_BuiltinStillAuthenticatesWithEnvAndToken(t *testing.T) t.Cleanup(func() { require.NoError(t, svc.Close()) }) require.NoError(t, svc.SaveSourceToken("nexusmods", "stored-db-key")) - registerSources(svc, t.TempDir()) + registerSources(svc, t.TempDir(), t.TempDir()) src, err := svc.GetSource("nexusmods") require.NoError(t, err) @@ -101,7 +101,7 @@ func TestRegisterSource_KeyResolutionPrecedence(t *testing.T) { mockAuthSource: mockAuthSource{id: "precedence-src", name: "Precedence Src"}, envKey: envVar, } - registerSource(svc, mock) + registerSource(svc, mock, t.TempDir()) assert.Equal(t, "env-value", mock.apiKey, "env var must take precedence over a stored DB token") }) @@ -115,12 +115,42 @@ func TestRegisterSource_KeyResolutionPrecedence(t *testing.T) { mockAuthSource: mockAuthSource{id: "precedence-src", name: "Precedence Src"}, envKey: envVar, } - registerSource(svc, mock) + registerSource(svc, mock, t.TempDir()) assert.Equal(t, "token-value", mock.apiKey, "stored token must apply when no env var is set") }) } +// recordingDataDirSource is a mockAuthSource that also implements the +// optional SetDataDir(string) setter (icarus.Icarus, #136 Task 13), so +// TestRegisterSource_WiresDataDir can pin that registerSource calls it with +// the resolved data directory - the same optional-setter pattern SetAPIKey +// already uses, just for a different capability. +type recordingDataDirSource struct { + mockAuthSource + dataDir string +} + +func (r *recordingDataDirSource) SetDataDir(dataDir string) { r.dataDir = dataDir } + +// TestRegisterSource_WiresDataDir pins that registerSource calls SetDataDir +// on a source that implements it, passing through the exact dataDir it was +// given - the seam icarus.Icarus.SetDataDir relies on to ever get a working +// dumps store outside of a test that constructs Icarus directly. +func TestRegisterSource_WiresDataDir(t *testing.T) { + svc, err := core.NewService(core.ServiceConfig{ + ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir(), + }) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + mock := &recordingDataDirSource{mockAuthSource: mockAuthSource{id: "data-dir-src", name: "Data Dir Src"}} + wantDataDir := t.TempDir() + registerSource(svc, mock, wantDataDir) + + assert.Equal(t, wantDataDir, mock.dataDir, "registerSource must pass its dataDir through to SetDataDir") +} + // TestRegisterSources_DerivedEnvKeyForCustom pins that a custom source with // no EnvKeyProvider still resolves its key via the derived LMM__API_KEY // convention (envKeyFor's fallback to envKeyForSourceID) through the unified @@ -150,7 +180,7 @@ manifest: `) t.Setenv("LMM_MY_CUSTOM_API_KEY", "custom-env-key") - registerSources(svc, cfgDir) + registerSources(svc, cfgDir, t.TempDir()) src, err := svc.GetSource("my-custom") require.NoError(t, err) @@ -184,7 +214,7 @@ directory: customSourceWarnOut = &warnBuf t.Cleanup(func() { customSourceWarnOut = nil }) - registerSources(svc, cfgDir) + registerSources(svc, cfgDir, t.TempDir()) src, err := svc.GetSource("nexusmods") require.NoError(t, err) @@ -330,7 +360,7 @@ directory: path: /this/path/should/not/exist/lmm-test-fixture `) // construction-failure branch: Validate passes, NewDirectory's os.Stat fails - registerCustomSources(svc, cfgDir) + registerCustomSources(svc, cfgDir, t.TempDir()) sources := svc.ListSources() byID := make(map[string]source.ModSource, len(sources)) diff --git a/docs/configuration.md b/docs/configuration.md index eb03632..b49d6ba 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -20,16 +20,17 @@ Defines moddable games. Each game is keyed by a unique slug (e.g. `skyrim-se`). ### Game options -| Option | Type | Required | Description | -| -------------- | ------ | -------- | ---------------------------------------------------------- | -| `name` | string | yes | Display name | -| `install_path` | string | yes | Game installation directory (supports `~`) | -| `mod_path` | string | yes | Directory where mods are deployed (supports `~`) | -| `sources` | map | yes | Source ID to game ID mapping (see below) | -| `link_method` | string | no | Override global link method: `symlink`, `hardlink`, `copy` | -| `cache_path` | string | no | Per-game cache directory override | -| `hooks` | object | no | Scripts to run around install/uninstall (see below) | -| `deploy_mode` | string | no | How to handle mod archives: `extract` (default) or `copy` | +| Option | Type | Required | Description | +| ---------------- | ------ | -------- | ------------------------------------------------------------------------------------------------ | +| `name` | string | yes | Display name | +| `install_path` | string | yes | Game installation directory (supports `~`) | +| `mod_path` | string | yes | Directory where mods are deployed (supports `~`) | +| `sources` | map | yes | Source ID to game ID mapping (see below) | +| `link_method` | string | no | Override global link method: `symlink`, `hardlink`, `copy` | +| `cache_path` | string | no | Per-game cache directory override | +| `hooks` | object | no | Scripts to run around install/uninstall (see below) | +| `deploy_mode` | string | no | How to handle mod archives: `extract` (default), `copy`, or `compile` | +| `data_dump_path` | string | no | Compile-mode only: local unpacked data.pak JSON tree, used instead of the hosted base-table dump | ### Hooks (games.yaml) @@ -57,6 +58,7 @@ The `deploy_mode` option controls how downloaded mod archives are handled: - **`extract`** (default): Archives are extracted to the mod path. Use for games where mods are loose files (e.g., Skyrim, Fallout). - **`copy`**: Archives are copied as-is to the mod path without extraction. Use for games that expect mod files to remain as archives (e.g., Minecraft `.jar` files, some Unity games). +- **`compile`**: The downloaded file is compiled into a new artifact before caching (currently Icarus only: an `.exmodz` diff is applied to the game's base data tables to produce a deployable `_P.pak`). Only sources that implement compiling support this mode. Optional `data_dump_path` points compilation at your own unpacked `data.pak` JSON tree instead of the hosted community dump; it must match the installed game version, and a mismatch is a hard error. Example: diff --git a/internal/core/service.go b/internal/core/service.go index 9eb89fd..db79a54 100644 --- a/internal/core/service.go +++ b/internal/core/service.go @@ -499,6 +499,33 @@ func (s *Service) DownloadModToCache(ctx context.Context, gameCache *cache.Cache return nil, err } defer os.RemoveAll(stagePath) //nolint:errcheck + + if game.DeployMode == domain.DeployCompile { + compiler, ok := src.(source.Compiler) + if !ok { + return nil, fmt.Errorf("source %q: game %q requires DeployCompile but source does not implement Compiler", src.ID(), game.ID) + } + basePakPath, err := resolveBasePak(game) + if err != nil { + return nil, err + } + // Unlike copyFileStreaming (which mkdirs its destination itself), + // Compile writes via unrealpak.Create - a bare os.Create - so + // stagePath must exist before it's called. + if err := os.MkdirAll(stagePath, 0755); err != nil { + return nil, fmt.Errorf("preparing compile staging: %w", err) + } + destName := compiledFileName(file.FileName) + destPath := filepath.Join(stagePath, destName) + if err := compiler.Compile(ctx, basePakPath, game.BaseDataPath, archivePath, destPath); err != nil { + return nil, fmt.Errorf("compiling mod: %w", err) + } + if err := commitStagedCacheWithMarker(cachePath, stagePath, file.ID, []string{destName}); err != nil { + return nil, err + } + return &DownloadModResult{FilesExtracted: 1, Checksum: downloadResult.Checksum}, nil + } + if game.DeployMode == domain.DeployCopy || !s.extractor.CanExtract(archivePath) { // Copy mode: game wants files as-is (e.g., Hytale .zip mods) // Or not an archive - just copy to cache. copyFileStreaming mkdirs @@ -893,6 +920,35 @@ func commitStagedCache(cachePath, stagePath string) error { return nil } +// resolveBasePak locates the currently-installed game's base pak for +// DeployCompile sources. v1 scope: Icarus only, one known pak filename +// pattern — extend this if a second DeployCompile-using game is ever added +// rather than generalizing speculatively now. The relative path below is +// Task 1's empirically-confirmed finding (docs/plans/icarus-pak-format-findings.md), +// recorded before this function was written, not an assumption made here: the +// JSON data tables live in Content/Data/data.pak, NOT in the Content/Paks +// pakchunks, which carry only cooked .uasset/.uexp assets and no JSON at all. +// +// Since rev3 this pak is no longer the source of base table *content* (that +// comes from the hosted dump — Task 12a); it is still required, because it is +// the only authority on which tables the installed game has and on which game +// week is installed. Its parent directory also locates Icarus/Config/version.json. +func resolveBasePak(game *domain.Game) (string, error) { + candidate := filepath.Join(game.InstallPath, "Icarus", "Content", "Data", "data.pak") + if _, err := os.Stat(candidate); err != nil { + return "", fmt.Errorf("locating base pak for %q: %w", game.ID, err) + } + return candidate, nil +} + +// compiledFileName turns a downloaded source filename into the cached +// output's name: same base name, .pak extension, matching Icarus's "_P.pak" +// override convention. +func compiledFileName(sourceFileName string) string { + base := strings.TrimSuffix(sourceFileName, filepath.Ext(sourceFileName)) + return base + "_P.pak" +} + // GetGame retrieves a game by ID func (s *Service) GetGame(gameID string) (*domain.Game, error) { game, ok := s.games[gameID] diff --git a/internal/core/service_icarus_compile_test.go b/internal/core/service_icarus_compile_test.go new file mode 100644 index 0000000..e9b1dc5 --- /dev/null +++ b/internal/core/service_icarus_compile_test.go @@ -0,0 +1,110 @@ +package core_test + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/core" + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/source" + "github.com/stretchr/testify/require" +) + +// fakeCompilerSource is a minimal ModSource that also implements +// source.Compiler, standing in for internal/source/icarus.Icarus (Tasks +// 8/13) without pulling that package into internal/core's tests — this test +// only needs to prove Service invokes Compile when DeployMode is +// DeployCompile, which Task 12 already tests in isolation. +type fakeCompilerSource struct { + downloadURL string + compileCalls int +} + +func (s *fakeCompilerSource) ID() string { return "fake-compiler" } +func (s *fakeCompilerSource) Name() string { return "Fake Compiler Source" } +func (s *fakeCompilerSource) AuthURL() string { return "" } +func (s *fakeCompilerSource) ExchangeToken(ctx context.Context, code string) (*source.Token, error) { + return nil, source.ErrNotSupported +} +func (s *fakeCompilerSource) Search(ctx context.Context, query source.SearchQuery) (source.SearchResult, error) { + return source.SearchResult{}, source.ErrNotSupported +} +func (s *fakeCompilerSource) GetMod(ctx context.Context, gameID, modID string) (*domain.Mod, error) { + return nil, source.ErrNotSupported +} +func (s *fakeCompilerSource) GetDependencies(ctx context.Context, mod *domain.Mod) ([]domain.ModReference, error) { + return nil, source.ErrNotSupported +} +func (s *fakeCompilerSource) GetModFiles(ctx context.Context, mod *domain.Mod) ([]domain.DownloadableFile, error) { + return nil, source.ErrNotSupported +} +func (s *fakeCompilerSource) GetDownloadURL(ctx context.Context, mod *domain.Mod, fileID string) (string, error) { + return s.downloadURL, nil +} +func (s *fakeCompilerSource) CheckUpdates(ctx context.Context, installed []domain.InstalledMod) ([]domain.Update, error) { + return nil, source.ErrNotSupported +} + +// Compile implements source.Compiler by copying the downloaded source file +// through unchanged — this test only asserts Service invoked it with the +// right arguments and used its output, not that it performs real PAK +// compilation (Task 12 covers that). +func (s *fakeCompilerSource) Compile(ctx context.Context, basePakPath, baseDataPath, sourceFilePath, outputPath string) error { + s.compileCalls++ + data, err := os.ReadFile(sourceFilePath) + if err != nil { + return err + } + return os.WriteFile(outputPath, data, 0o644) +} + +var ( + _ source.ModSource = (*fakeCompilerSource)(nil) + _ source.Compiler = (*fakeCompilerSource)(nil) +) + +func TestDownloadMod_DeployCompile_InvokesCompiler(t *testing.T) { + dlSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("fake-exmodz-bytes")) + })) + defer dlSrv.Close() + + installDir := t.TempDir() + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + require.NoError(t, os.WriteFile(basePak, []byte("fake-base-pak"), 0o644)) + + cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} + svc, err := core.NewService(cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + src := &fakeCompilerSource{downloadURL: dlSrv.URL} + svc.RegisterSource(src) + + game := &domain.Game{ID: "icarus", InstallPath: installDir, ModPath: t.TempDir(), DeployMode: domain.DeployCompile} + require.NoError(t, svc.AddGame(game)) + + mod := &domain.Mod{ID: "bear-mount", SourceID: "fake-compiler", GameID: "icarus", Version: "3.3"} + file := &domain.DownloadableFile{ID: "exmodz", FileName: "Bear_Mount.exmodz"} + + result, err := svc.DownloadMod(context.Background(), "fake-compiler", game, mod, file, nil) + require.NoError(t, err) + require.Equal(t, 1, result.FilesExtracted) + require.Equal(t, 1, src.compileCalls) + + gameCache := svc.GetGameCache(game) + require.True(t, gameCache.Exists(game.ID, mod.SourceID, mod.ID, mod.Version)) + files, err := gameCache.ListFiles(game.ID, mod.SourceID, mod.ID, mod.Version) + require.NoError(t, err) + require.Len(t, files, 1) + require.Equal(t, "Bear_Mount_P.pak", files[0]) + + data, err := os.ReadFile(gameCache.GetFilePath(game.ID, mod.SourceID, mod.ID, mod.Version, files[0])) + require.NoError(t, err) + require.Equal(t, "fake-exmodz-bytes", string(data)) +} diff --git a/internal/domain/game.go b/internal/domain/game.go index 871ca03..6166266 100644 --- a/internal/domain/game.go +++ b/internal/domain/game.go @@ -46,6 +46,9 @@ type Game struct { CachePath string // Optional: custom cache path for this game's mods Hooks GameHooks // Optional: hooks for install/uninstall operations DeployMode DeployMode // How to handle downloaded files (extract vs copy) + // BaseDataPath is optional: a directory holding an unpacked data.pak JSON + // tree, used instead of fetching the hosted base-table dump (compile games only) + BaseDataPath string } // DeployMode determines how downloaded mod archives are handled @@ -54,6 +57,7 @@ type DeployMode int const ( DeployExtract DeployMode = iota // Default: extract archives to mod path DeployCopy // Copy files as-is (for games like Hytale where .zip IS the mod) + DeployCompile // Compile downloaded file into a new artifact before caching (Icarus .exmodz -> .pak) ) func (m DeployMode) String() string { @@ -62,6 +66,8 @@ func (m DeployMode) String() string { return "extract" case DeployCopy: return "copy" + case DeployCompile: + return "compile" default: return "extract" } @@ -72,6 +78,8 @@ func ParseDeployMode(s string) DeployMode { switch s { case "copy": return DeployCopy + case "compile": + return DeployCompile default: return DeployExtract } diff --git a/internal/domain/game_test.go b/internal/domain/game_test.go index f170718..a3ec13b 100644 --- a/internal/domain/game_test.go +++ b/internal/domain/game_test.go @@ -58,6 +58,7 @@ func TestDeployMode_String(t *testing.T) { }{ {"extract", DeployExtract, "extract"}, {"copy", DeployCopy, "copy"}, + {"compile", DeployCompile, "compile"}, // Unlike LinkMethod.String, the default branch here also returns // "extract" rather than "unknown" for out-of-range values. {"unknown value falls back to extract", DeployMode(99), "extract"}, @@ -77,6 +78,7 @@ func TestParseDeployMode(t *testing.T) { want DeployMode }{ {"copy", "copy", DeployCopy}, + {"compile", "compile", DeployCompile}, {"extract explicit", "extract", DeployExtract}, {"empty defaults to extract", "", DeployExtract}, {"unknown defaults to extract", "bogus", DeployExtract}, diff --git a/internal/source/icarus/icarus.go b/internal/source/icarus/icarus.go index 794fddb..4d63a4d 100644 --- a/internal/source/icarus/icarus.go +++ b/internal/source/icarus/icarus.go @@ -6,6 +6,7 @@ import ( "net/http" "net/url" "path" + "path/filepath" "strings" "github.com/DonovanMods/linux-mod-manager/internal/domain" @@ -19,6 +20,7 @@ const gameID = "icarus" // API described in docs/plans/2026-07-29-icarus-exmod-pak-research.md. type Icarus struct { firestore *firestoreClient + dumps *DumpStore // nil until SetDataDir is called } // New constructs an Icarus source. projectID is the Firestore project ID @@ -29,11 +31,36 @@ func New(httpClient *http.Client, projectID string) *Icarus { return &Icarus{firestore: newFirestoreClient(projectID, httpClient)} } +// SetDataDir wires the base-table dump store's cache directory once the +// service's data directory is known. This is a post-construction setter +// rather than a New parameter because Task 8 froze New(httpClient, projectID) +// at exactly those two params — Task 9's call site already depends on that +// signature — so the data dir arrives the same way API keys do: an optional +// setter the registration pipeline calls when present (cmd/lmm/root.go's +// registerSource, mirroring its existing SetAPIKey wiring). +func (s *Icarus) SetDataDir(dataDir string) { + s.dumps = newDumpStore(filepath.Join(dataDir, "icarus", "datadump"), s.firestore.httpClient) +} + var ( _ source.ModSource = (*Icarus)(nil) _ source.CapabilityReporter = (*Icarus)(nil) + _ source.Compiler = (*Icarus)(nil) ) +// Compile implements source.Compiler by delegating to the package-level +// Compile function (Task 12) — basePakPath/baseDataPath/sourceFilePath/ +// outputPath map directly onto Compile's basePakPath/localDumpDir/exmodzPath/ +// outputPakPath parameters. The base-table dump store (Task 12a) is supplied +// from the source itself; the per-game dump-directory override arrives as +// baseDataPath, since only the caller has the game's config. +func (s *Icarus) Compile(ctx context.Context, basePakPath, baseDataPath, sourceFilePath, outputPath string) error { + if s.dumps == nil { + return fmt.Errorf("source %q: not initialized with a data directory (SetDataDir was never called)", s.ID()) + } + return Compile(ctx, s.dumps, basePakPath, baseDataPath, sourceFilePath, outputPath) +} + func (s *Icarus) ID() string { return "icarus" } func (s *Icarus) Name() string { return "Icarus (Project Daedalus)" } diff --git a/internal/source/icarus/icarus_test.go b/internal/source/icarus/icarus_test.go index 7e2780d..f21b877 100644 --- a/internal/source/icarus/icarus_test.go +++ b/internal/source/icarus/icarus_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strings" "testing" "github.com/DonovanMods/linux-mod-manager/internal/domain" @@ -83,3 +84,30 @@ func TestIcarus_GetModFiles_ReturnsExmodzAndPak(t *testing.T) { t.Error("single file should be marked primary") } } + +// TestIcarus_Compile_WithoutDataDir_FailsLoudly pins that a source +// constructed via New but never wired with SetDataDir (e.g. a registration +// path that forgets the optional-setter call) fails loudly instead of +// panicking on a nil dumps store. +func TestIcarus_Compile_WithoutDataDir_FailsLoudly(t *testing.T) { + src := New(nil, "test-project") + + err := src.Compile(context.Background(), "/base.pak", "", "/mod.exmodz", "/out.pak") + if err == nil { + t.Fatal("Compile: expected an error when SetDataDir was never called") + } + if !strings.Contains(err.Error(), "SetDataDir") { + t.Errorf("Compile error = %q, want it to mention SetDataDir", err.Error()) + } +} + +// TestIcarus_SetDataDir_ConstructsDumpStore pins that SetDataDir wires a +// non-nil dumps store, so a real registration call unblocks Compile. +func TestIcarus_SetDataDir_ConstructsDumpStore(t *testing.T) { + src := New(nil, "test-project") + src.SetDataDir(t.TempDir()) + + if src.dumps == nil { + t.Fatal("SetDataDir: dumps store still nil") + } +} diff --git a/internal/source/source.go b/internal/source/source.go index c3261f6..622d5f6 100644 --- a/internal/source/source.go +++ b/internal/source/source.go @@ -142,3 +142,19 @@ func CapabilitiesOf(src ModSource) Capabilities { type DownloadHeaderProvider interface { DownloadHeaders(fileURL string) map[string]string } + +// Compiler is implemented by sources whose downloaded files need +// transforming into a different artifact before deployment (Icarus's +// .exmodz -> .pak). Service consults it, when DeployMode is DeployCompile, +// after downloading but before committing the file to cache — the result +// replaces the downloaded file in cache, so everything downstream (Install, +// the linker) treats it exactly like a DeployCopy file. +// +// basePakPath and baseDataPath are both resolved by the caller from the game's +// config: basePakPath from game.InstallPath, baseDataPath from the game's +// optional data_dump_path ("" when unset — see Step 6b). sourceFilePath is the +// just-downloaded file; outputPath is where the compiled result must be +// written. +type Compiler interface { + Compile(ctx context.Context, basePakPath, baseDataPath, sourceFilePath, outputPath string) error +} diff --git a/internal/storage/config/games.go b/internal/storage/config/games.go index ee7d142..8860115 100644 --- a/internal/storage/config/games.go +++ b/internal/storage/config/games.go @@ -49,14 +49,15 @@ type GameHooksYAML struct { // GameConfig is the YAML representation of a game type GameConfig struct { - Name string `yaml:"name"` - InstallPath string `yaml:"install_path"` - ModPath string `yaml:"mod_path"` - Sources map[string]string `yaml:"sources"` - LinkMethod string `yaml:"link_method,omitempty"` - CachePath string `yaml:"cache_path,omitempty"` - Hooks GameHooksYAML `yaml:"hooks,omitempty"` - DeployMode string `yaml:"deploy_mode,omitempty"` + Name string `yaml:"name"` + InstallPath string `yaml:"install_path"` + ModPath string `yaml:"mod_path"` + Sources map[string]string `yaml:"sources"` + LinkMethod string `yaml:"link_method,omitempty"` + CachePath string `yaml:"cache_path,omitempty"` + Hooks GameHooksYAML `yaml:"hooks,omitempty"` + DeployMode string `yaml:"deploy_mode,omitempty"` + BaseDataPath string `yaml:"data_dump_path,omitempty"` } // GamesFile is the top-level games.yaml structure @@ -97,6 +98,7 @@ func loadGamesLocked(configDir string) (map[string]*domain.Game, error) { LinkMethodExplicit: cfg.LinkMethod != "", CachePath: ExpandPath(cfg.CachePath), DeployMode: domain.ParseDeployMode(cfg.DeployMode), + BaseDataPath: ExpandPath(cfg.BaseDataPath), Hooks: domain.GameHooks{ Install: domain.HookConfig{ BeforeAll: ExpandPath(cfg.Hooks.Install.BeforeAll), @@ -134,11 +136,12 @@ func saveGamesLocked(configDir string, games map[string]*domain.Game) error { for id, game := range games { cfg := GameConfig{ - Name: game.Name, - InstallPath: game.InstallPath, - ModPath: game.ModPath, - Sources: game.SourceIDs, - CachePath: game.CachePath, + Name: game.Name, + InstallPath: game.InstallPath, + ModPath: game.ModPath, + Sources: game.SourceIDs, + CachePath: game.CachePath, + BaseDataPath: game.BaseDataPath, Hooks: GameHooksYAML{ Install: HookConfigYAML{ BeforeAll: game.Hooks.Install.BeforeAll, diff --git a/internal/storage/config/games_test.go b/internal/storage/config/games_test.go new file mode 100644 index 0000000..30fe64a --- /dev/null +++ b/internal/storage/config/games_test.go @@ -0,0 +1,24 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func TestLoadGames_DataDumpPath(t *testing.T) { + dir := t.TempDir() + yaml := "games:\n icarus:\n name: Icarus\n install_path: /games/icarus\n" + + " mod_path: /games/icarus/mods\n data_dump_path: /dumps/week243\n" + if err := os.WriteFile(filepath.Join(dir, "games.yaml"), []byte(yaml), 0o644); err != nil { + t.Fatal(err) + } + + games, err := LoadGames(dir) + if err != nil { + t.Fatalf("LoadGames: %v", err) + } + if got := games["icarus"].BaseDataPath; got != "/dumps/week243" { + t.Errorf("BaseDataPath = %q, want /dumps/week243", got) + } +} From 74f3af58a408b9c99d20538bb9d6028955f1e55d Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Fri, 31 Jul 2026 21:22:24 -0400 Subject: [PATCH 16/96] fix: gate exmod compile branch on file, not just game (#136) Review fix round 1: a DeployCompile game's compile branch ran on every downloaded file, but icarus.GetModFiles can also serve a prebuilt .pak alongside the .exmodz diff - routing a .pak through Compile fails loudly since it isn't a zip, making plain-pak Icarus mods permanently uninstallable. Gate the branch on file.FileName ending in ".exmodz" (case-insensitive) as well as DeployMode; any other file (notably .pak) falls through to the existing extract/copy path unchanged. No new config. Co-Authored-By: Claude Sonnet 5 --- internal/core/service.go | 13 +- internal/core/service_icarus_compile_test.go | 142 +++++++++++++++++++ 2 files changed, 154 insertions(+), 1 deletion(-) diff --git a/internal/core/service.go b/internal/core/service.go index db79a54..ad15736 100644 --- a/internal/core/service.go +++ b/internal/core/service.go @@ -500,7 +500,7 @@ func (s *Service) DownloadModToCache(ctx context.Context, gameCache *cache.Cache } defer os.RemoveAll(stagePath) //nolint:errcheck - if game.DeployMode == domain.DeployCompile { + if game.DeployMode == domain.DeployCompile && isExmodzFile(file.FileName) { compiler, ok := src.(source.Compiler) if !ok { return nil, fmt.Errorf("source %q: game %q requires DeployCompile but source does not implement Compiler", src.ID(), game.ID) @@ -920,6 +920,17 @@ func commitStagedCache(cachePath, stagePath string) error { return nil } +// isExmodzFile reports whether fileName is a compile-eligible archive +// (case-insensitive ".exmodz" suffix). DeployCompile games can also serve +// plain, already-built ".pak" files (icarus.GetModFiles enumerates "pak" +// before "exmodz") - those must NOT be routed through Compile, which expects +// an .exmodz diff (#136 review, Task 13 fix round 1): a prebuilt pak falls +// through to the pre-compile extract/copy logic unchanged, exactly as if +// DeployMode were not DeployCompile at all. +func isExmodzFile(fileName string) bool { + return strings.HasSuffix(strings.ToLower(fileName), ".exmodz") +} + // resolveBasePak locates the currently-installed game's base pak for // DeployCompile sources. v1 scope: Icarus only, one known pak filename // pattern — extend this if a second DeployCompile-using game is ever added diff --git a/internal/core/service_icarus_compile_test.go b/internal/core/service_icarus_compile_test.go index e9b1dc5..3a7ff2a 100644 --- a/internal/core/service_icarus_compile_test.go +++ b/internal/core/service_icarus_compile_test.go @@ -108,3 +108,145 @@ func TestDownloadMod_DeployCompile_InvokesCompiler(t *testing.T) { require.NoError(t, err) require.Equal(t, "fake-exmodz-bytes", string(data)) } + +// newCompileTestGame builds a DeployCompile game backed by fakeCompilerSource, +// serving dlBody for every download - shared setup for +// TestDownloadMod_DeployCompile_RoutesPerFile's cases. +func newCompileTestGame(t *testing.T, dlBody string) (*core.Service, *fakeCompilerSource, *domain.Game) { + t.Helper() + + dlSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(dlBody)) + })) + t.Cleanup(dlSrv.Close) + + installDir := t.TempDir() + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + require.NoError(t, os.WriteFile(basePak, []byte("fake-base-pak"), 0o644)) + + cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} + svc, err := core.NewService(cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + src := &fakeCompilerSource{downloadURL: dlSrv.URL} + svc.RegisterSource(src) + + game := &domain.Game{ID: "icarus", InstallPath: installDir, ModPath: t.TempDir(), DeployMode: domain.DeployCompile} + require.NoError(t, svc.AddGame(game)) + + return svc, src, game +} + +// TestDownloadMod_DeployCompile_RoutesPerFile pins the fix-round-1 gap: a +// DeployCompile game's compile branch must key off the FILE (".exmodz" +// suffix, case-insensitive), not the game alone - icarus.GetModFiles can +// serve a mod's already-built ".pak" alongside its ".exmodz" diff, and a pak +// routed into Compile fails (it isn't a zip ParseExmodz can read). +func TestDownloadMod_DeployCompile_RoutesPerFile(t *testing.T) { + tests := []struct { + name string + fileName string + wantCompiled bool + wantCachedName string + }{ + {"exmodz file takes the compile branch", "Bear_Mount.exmodz", true, "Bear_Mount_P.pak"}, + {"EXMODZ file takes the compile branch case-insensitively", "Bear_Mount.EXMODZ", true, "Bear_Mount_P.pak"}, + {"pak file skips the compiler entirely", "Bear_Mount.pak", false, "Bear_Mount.pak"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + const body = "fake-download-bytes" + svc, src, game := newCompileTestGame(t, body) + + mod := &domain.Mod{ID: "bear-mount", SourceID: "fake-compiler", GameID: "icarus", Version: "3.3"} + file := &domain.DownloadableFile{ID: "the-file", FileName: tt.fileName} + + result, err := svc.DownloadMod(context.Background(), "fake-compiler", game, mod, file, nil) + require.NoError(t, err) + require.Equal(t, 1, result.FilesExtracted) + + wantCompileCalls := 0 + if tt.wantCompiled { + wantCompileCalls = 1 + } + require.Equal(t, wantCompileCalls, src.compileCalls) + + gameCache := svc.GetGameCache(game) + files, err := gameCache.ListFiles(game.ID, mod.SourceID, mod.ID, mod.Version) + require.NoError(t, err) + require.Equal(t, []string{tt.wantCachedName}, files) + + data, err := os.ReadFile(gameCache.GetFilePath(game.ID, mod.SourceID, mod.ID, mod.Version, tt.wantCachedName)) + require.NoError(t, err) + require.Equal(t, body, string(data)) + }) + } + + // Regression proof for the ".pak" case above: rather than only asserting + // "Compile wasn't called" (a fake-tautology that would also pass if + // routing were broken some other way), this proves a DeployCompile game + // handling a plain ".pak" produces EXACTLY what a DeployExtract game + // produces for the identical file through the identical source - the + // genuine pre-Task-13 extract/copy path, byte-for-byte. + t.Run("pak file on a compile-mode game matches a non-compile game byte-for-byte", func(t *testing.T) { + const body = "fake-pak-bytes" + mod := &domain.Mod{ID: "bear-mount", SourceID: "fake-compiler", GameID: "icarus", Version: "3.3"} + file := &domain.DownloadableFile{ID: "pak", FileName: "Bear_Mount.pak"} + + compileSvc, compileSrc, compileGame := newCompileTestGame(t, body) + compileResult, err := compileSvc.DownloadMod(context.Background(), "fake-compiler", compileGame, mod, file, nil) + require.NoError(t, err) + + extractSvc, extractSrc, extractGame := newCompileTestGame(t, body) + extractGame.DeployMode = domain.DeployExtract + extractResult, err := extractSvc.DownloadMod(context.Background(), "fake-compiler", extractGame, mod, file, nil) + require.NoError(t, err) + + require.Equal(t, 0, compileSrc.compileCalls) + require.Equal(t, 0, extractSrc.compileCalls) + require.Equal(t, extractResult, compileResult) + + compileFiles, err := compileSvc.GetGameCache(compileGame).ListFiles(compileGame.ID, mod.SourceID, mod.ID, mod.Version) + require.NoError(t, err) + extractFiles, err := extractSvc.GetGameCache(extractGame).ListFiles(extractGame.ID, mod.SourceID, mod.ID, mod.Version) + require.NoError(t, err) + require.Equal(t, extractFiles, compileFiles) + + compileData, err := os.ReadFile(compileSvc.GetGameCache(compileGame).GetFilePath(compileGame.ID, mod.SourceID, mod.ID, mod.Version, compileFiles[0])) + require.NoError(t, err) + extractData, err := os.ReadFile(extractSvc.GetGameCache(extractGame).GetFilePath(extractGame.ID, mod.SourceID, mod.ID, mod.Version, extractFiles[0])) + require.NoError(t, err) + require.Equal(t, extractData, compileData) + }) +} + +// TestDownloadMod_DeployCompile_MixedFileMod pins that a single mod shipping +// both a prebuilt ".pak" and an ".exmodz" diff (icarus.GetModFiles's "pak" +// then "exmodz" enumeration, neither marked primary when both are present) +// gets each file routed independently within the same DeployCompile game: +// one DownloadMod call per DownloadableFile, exactly as the real CLI/TUI +// download flow drives it. +func TestDownloadMod_DeployCompile_MixedFileMod(t *testing.T) { + svc, src, game := newCompileTestGame(t, "fake-bytes") + mod := &domain.Mod{ID: "bear-mount", SourceID: "fake-compiler", GameID: "icarus", Version: "3.3"} + + exmodzFile := &domain.DownloadableFile{ID: "exmodz", FileName: "Bear_Mount.exmodz"} + pakFile := &domain.DownloadableFile{ID: "pak", FileName: "Bear_Mount.pak"} + + _, err := svc.DownloadMod(context.Background(), "fake-compiler", game, mod, exmodzFile, nil) + require.NoError(t, err) + require.Equal(t, 1, src.compileCalls, "exmodz file must compile") + + _, err = svc.DownloadMod(context.Background(), "fake-compiler", game, mod, pakFile, nil) + require.NoError(t, err) + require.Equal(t, 1, src.compileCalls, "pak file must not trigger a second compile") + + gameCache := svc.GetGameCache(game) + files, err := gameCache.ListFiles(game.ID, mod.SourceID, mod.ID, mod.Version) + require.NoError(t, err) + require.ElementsMatch(t, []string{"Bear_Mount_P.pak", "Bear_Mount.pak"}, files, + "both the compiled exmodz output and the untouched pak must be cached") +} From 2578b372f562d5cf4b7b8fb40a8e6c51f9d24d6d Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Fri, 31 Jul 2026 21:27:29 -0400 Subject: [PATCH 17/96] docs: add CHANGELOG entry for Icarus mod support (#136) --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 703923d..7b6ca11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **Icarus built-in mod source** (`internal/source/icarus`): a public, unauthenticated Firestore-backed catalog (Project Daedalus) — `lmm search`/`install`/`update` work against it like NexusMods/CurseForge. A `.exmodz` mod file now compiles into a deployable `_P.pak` at download time via a new, game-agnostic `internal/unrealpak` PAK reader/writer and the new `deploy_mode: compile` game setting; a plain `.pak` file from the same catalog is unaffected and deploys through the existing extract/copy pipeline unchanged. An optional per-game `data_dump_path` points compilation at your own unpacked `data.pak` JSON tree instead of fetching the hosted community base-table dump — both are `games.yaml`-only settings, with no new CLI flag or TUI screen (#136) + ## [1.27.1] - 2026-07-30 ### Fixed From 6cac5bdd40d5e4e448fcab352ee8e21e9073557d Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Fri, 31 Jul 2026 21:36:59 -0400 Subject: [PATCH 18/96] chore: silence errcheck on intentional Close defers (#136) Final-review finding I1: trunk check reported 6 new errcheck findings for deferred Close() calls whose error is intentionally ignored (the repo idiom elsewhere, e.g. datadump.go/service.go, already appends //nolint:errcheck to these). Annotated the flagged sites in compile.go and compile_test.go. Re-running trunk check after that fix surfaced 2 more unannotated defers of the same class in reader_test.go:222 and roundtrip_test.go:36 (both files are wholly new to this branch, so every finding in them counts as branch- introduced, not pre-existing debt) -- annotated those too so the branch carries zero new lint issues, not just the six named in final-review.md. trunk check: 0 new issues (14 pre-existing, out of scope, unchanged). --- internal/source/icarus/compile.go | 2 +- internal/source/icarus/compile_test.go | 4 ++-- internal/unrealpak/reader_test.go | 8 ++++---- internal/unrealpak/roundtrip_test.go | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/source/icarus/compile.go b/internal/source/icarus/compile.go index 384cb25..49929e6 100644 --- a/internal/source/icarus/compile.go +++ b/internal/source/icarus/compile.go @@ -41,7 +41,7 @@ func Compile(ctx context.Context, dumps *DumpStore, basePakPath, localDumpDir, e if err != nil { return fmt.Errorf("icarus: opening base pak %s: %w", basePakPath, err) } - defer base.Close() + defer base.Close() //nolint:errcheck // Loaded and validated before anything is written, so a week mismatch or // an offline machine fails before a half-built pak exists on disk. diff --git a/internal/source/icarus/compile_test.go b/internal/source/icarus/compile_test.go index 6614c77..51a9560 100644 --- a/internal/source/icarus/compile_test.go +++ b/internal/source/icarus/compile_test.go @@ -76,7 +76,7 @@ func TestCompile_AppliesDiffAndBundlesAssets(t *testing.T) { if err != nil { t.Fatalf("opening compiled output: %v", err) } - defer r.Close() + defer r.Close() //nolint:errcheck patched, err := r.ReadFile("AI/D_AIGrowth.json") if err != nil { @@ -118,7 +118,7 @@ func TestCompile_SkipsEndOfModSentinelRow(t *testing.T) { if err != nil { t.Fatalf("opening compiled output: %v", err) } - defer r.Close() + defer r.Close() //nolint:errcheck patched, err := r.ReadFile("AI/D_AIGrowth.json") if err != nil { t.Fatalf("ReadFile patched data table: %v", err) diff --git a/internal/unrealpak/reader_test.go b/internal/unrealpak/reader_test.go index fedf7f4..c6c17e6 100644 --- a/internal/unrealpak/reader_test.go +++ b/internal/unrealpak/reader_test.go @@ -109,7 +109,7 @@ func TestReader_Open_ListsFiles(t *testing.T) { if err != nil { t.Fatalf("Open: %v", err) } - defer r.Close() + defer r.Close() //nolint:errcheck files := r.Files() if len(files) != 1 { @@ -131,7 +131,7 @@ func TestReader_Open_RootLevelFile(t *testing.T) { if err != nil { t.Fatalf("Open: %v", err) } - defer r.Close() + defer r.Close() //nolint:errcheck files := r.Files() if len(files) != 1 || files[0].Path != "x.json" { @@ -196,7 +196,7 @@ func TestReader_ReadFile(t *testing.T) { if err != nil { t.Fatalf("Open: %v", err) } - defer r.Close() + defer r.Close() //nolint:errcheck got, err := r.ReadFile("Icarus/Content/Data/Test.json") if err != nil { @@ -219,7 +219,7 @@ func TestReader_ReadFile_RejectsCompressedEntry(t *testing.T) { if err != nil { t.Fatalf("Open: %v", err) } - defer r.Close() + defer r.Close() //nolint:errcheck // Enumeration must still work — the reader lists compressed entries. if files := r.Files(); len(files) != 1 || files[0].Path != name { diff --git a/internal/unrealpak/roundtrip_test.go b/internal/unrealpak/roundtrip_test.go index 2d0b9db..53b7b31 100644 --- a/internal/unrealpak/roundtrip_test.go +++ b/internal/unrealpak/roundtrip_test.go @@ -33,7 +33,7 @@ func TestRoundTrip_WriteThenRead(t *testing.T) { if err != nil { t.Fatalf("Open: %v", err) } - defer r.Close() + defer r.Close() //nolint:errcheck got := r.Files() if len(got) != len(files) { From 4d905f51455c445e79a27fcdcbf3d910599dd715 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Fri, 31 Jul 2026 21:49:55 -0400 Subject: [PATCH 19/96] fix: harden size validation, token escaping, and writer cleanup (review) (#136) PR #171 Copilot review fixes: - unrealpak: readRegion and Reader.ReadFile now validate a size field (non-negative, <= the pak file's own size) before using it to size an allocation, via a shared validateAllocSize helper. Reader gains a fileSize field, threaded through Open/parseIndex/readRegion. Closes the Task 2/3 deferred-minor 'unvalidated size pre-check allocation'. - unrealpak: cursor.take's already-err-latched early-return path now clamps n to >=0 before make(), matching the fresh-error branch a few lines below -- a corrupted length field could otherwise panic there. - icarus: firestore_client.listCollection now url.QueryEscape()s nextPageToken before appending it to the request URL, instead of concatenating the server-issued token raw. Closes Task 7's deferred minor. - icarus: Compile's error-path cleanup defer now closes the unrealpak.Writer (error ignored -- the partial output is about to be deleted anyway) before os.Remove, so the fd never leaks and the remove works on platforms that refuse to delete a still-open file. Success path unchanged. New tests: TestValidateAllocSize, TestReadRegion_RejectsInvalidSizeOrOffsetBeforeReading, TestReader_ReadFile_RejectsInvalidSizeField, TestCursor_Take_ClampsNegativeLengthOnErrLatchedPath, TestFirestoreClient_ListCollection_EscapesPageToken. --- internal/source/icarus/compile.go | 6 +- internal/source/icarus/firestore_client.go | 7 +- .../source/icarus/firestore_client_test.go | 42 +++++++ internal/unrealpak/reader.go | 58 ++++++--- internal/unrealpak/reader_test.go | 117 ++++++++++++++++++ 5 files changed, 211 insertions(+), 19 deletions(-) diff --git a/internal/source/icarus/compile.go b/internal/source/icarus/compile.go index 49929e6..4781cc8 100644 --- a/internal/source/icarus/compile.go +++ b/internal/source/icarus/compile.go @@ -60,11 +60,15 @@ func Compile(ctx context.Context, dumps *DumpStore, basePakPath, localDumpDir, e // no way to abort without finalizing (Close always serializes and writes // whatever was buffered), so removing the file is the only way to keep // the fail-loud-and-clean contract on this path; the success path - // (err == nil here) is untouched. + // (err == nil here) is untouched. The writer is closed (best-effort, + // error ignored — whatever it wrote is about to be deleted anyway) + // before the remove so the fd never leaks and the remove itself works on + // platforms (Windows) that refuse to delete a still-open file. defer func() { if err == nil { return } + _ = out.Close() //nolint:errcheck if rmErr := os.Remove(outputPakPath); rmErr != nil && !os.IsNotExist(rmErr) { err = fmt.Errorf("%w (additionally, removing partial output %s failed: %v)", err, outputPakPath, rmErr) } diff --git a/internal/source/icarus/firestore_client.go b/internal/source/icarus/firestore_client.go index 07a7e6d..8fb8da7 100644 --- a/internal/source/icarus/firestore_client.go +++ b/internal/source/icarus/firestore_client.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" "strings" ) @@ -42,9 +43,9 @@ func (c *firestoreClient) listCollection(ctx context.Context, collection string) var all []firestoreDoc pageToken := "" for { - url := fmt.Sprintf("%s/%s?pageSize=200", c.documentsURL(), collection) + reqURL := fmt.Sprintf("%s/%s?pageSize=200", c.documentsURL(), collection) if pageToken != "" { - url += "&pageToken=" + pageToken + reqURL += "&pageToken=" + url.QueryEscape(pageToken) } var page struct { Documents []struct { @@ -53,7 +54,7 @@ func (c *firestoreClient) listCollection(ctx context.Context, collection string) } `json:"documents"` NextPageToken string `json:"nextPageToken"` } - if err := c.getJSON(ctx, url, &page); err != nil { + if err := c.getJSON(ctx, reqURL, &page); err != nil { return nil, fmt.Errorf("listing %s: %w", collection, err) } for _, d := range page.Documents { diff --git a/internal/source/icarus/firestore_client_test.go b/internal/source/icarus/firestore_client_test.go index 4fa5cec..6b5bad7 100644 --- a/internal/source/icarus/firestore_client_test.go +++ b/internal/source/icarus/firestore_client_test.go @@ -51,6 +51,48 @@ func TestFirestoreClient_ListCollection_Paginates(t *testing.T) { } } +// nextPageToken is opaque server-issued data, not guaranteed URL-safe as-is; +// listCollection must query-escape it before appending it to the request URL +// rather than concatenating it raw. Round-trips a token containing +// characters ("&", "=", "+", "/", "?") that would corrupt the query string +// if not escaped, and asserts the mock server decodes it back to the exact +// original value. +func TestFirestoreClient_ListCollection_EscapesPageToken(t *testing.T) { + const rawToken = "page&two=x+y/z?w" + pages := []map[string]any{ + { + "documents": []map[string]any{ + {"name": "projects/p/databases/(default)/documents/mods/abc", "fields": map[string]any{}}, + }, + "nextPageToken": rawToken, + }, + { + "documents": []map[string]any{}, + }, + } + callCount := 0 + var gotPageToken string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if callCount == 1 { + gotPageToken = r.URL.Query().Get("pageToken") + } + page := pages[callCount] + callCount++ + json.NewEncoder(w).Encode(page) //nolint:errcheck + })) + defer srv.Close() + + c := newFirestoreClient("test-project", srv.Client()) + c.baseURL = srv.URL + + if _, err := c.listCollection(context.Background(), "mods"); err != nil { + t.Fatalf("listCollection: %v", err) + } + if gotPageToken != rawToken { + t.Errorf("server decoded pageToken = %q, want %q (round-trip through query-escaping)", gotPageToken, rawToken) + } +} + func TestFirestoreClient_GetDocument_NotFound(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) diff --git a/internal/unrealpak/reader.go b/internal/unrealpak/reader.go index e2c2c2e..a77eceb 100644 --- a/internal/unrealpak/reader.go +++ b/internal/unrealpak/reader.go @@ -13,8 +13,9 @@ import ( // Reader provides read access to an uncompressed, unencrypted UE4-range pak. type Reader struct { - f *os.File - entries []readerEntry + f *os.File + entries []readerEntry + fileSize int64 // total size of the underlying file, for validateAllocSize } type readerEntry struct { @@ -37,7 +38,9 @@ func Open(path string) (*Reader, error) { return nil, fmt.Errorf("unrealpak: stat %s: %w", path, err) } - ft, err := readFooter(f, info.Size()) + fileSize := info.Size() + + ft, err := readFooter(f, fileSize) if err != nil { f.Close() //nolint:errcheck return nil, err @@ -47,19 +50,33 @@ func Open(path string) (*Reader, error) { return nil, fmt.Errorf("unrealpak: %s: %w: encrypted index", path, ErrUnsupportedFormat) } - indexBuf, err := readRegion(f, ft.indexOffset, ft.indexSize, ft.indexHash) + indexBuf, err := readRegion(f, ft.indexOffset, ft.indexSize, fileSize, ft.indexHash) if err != nil { f.Close() //nolint:errcheck return nil, fmt.Errorf("unrealpak: %s: primary index: %w", path, err) } - entries, err := parseIndex(f, indexBuf) + entries, err := parseIndex(f, indexBuf, fileSize) if err != nil { f.Close() //nolint:errcheck return nil, fmt.Errorf("unrealpak: %s: parsing index: %w", path, err) } - return &Reader{f: f, entries: entries}, nil + return &Reader{f: f, entries: entries, fileSize: fileSize}, nil +} + +// validateAllocSize checks a length field read from pak data before it is +// used to size a make([]byte, ...) allocation: it must be non-negative and +// cannot exceed the pak file's own size — no genuine region or payload can be +// larger than the file that contains it. A field outside that range is +// corruption or a layout this package doesn't understand, never something to +// allocate for (a 64-bit size field with its top bit set becomes negative +// once cast to int64, which is exactly the case this exists to catch). +func validateAllocSize(size, fileSize int64) (int, error) { + if size < 0 || size > fileSize { + return 0, fmt.Errorf("%w: size field %d is invalid for a %d-byte pak", ErrUnsupportedFormat, size, fileSize) + } + return int(size), nil } // readRegion reads size bytes at offset and verifies them against want. Every @@ -67,11 +84,15 @@ func Open(path string) (*Reader, error) { // primary index, and the primary index covers each sub-index. All three gates // are enforced — a mismatch is corruption or an unrecognized layout, never // something to parse through. -func readRegion(r io.ReaderAt, offset, size int64, want [20]byte) ([]byte, error) { - if offset < 0 || size < 0 { - return nil, fmt.Errorf("%w: negative region offset/size", ErrUnsupportedFormat) +func readRegion(r io.ReaderAt, offset, size, fileSize int64, want [20]byte) ([]byte, error) { + if offset < 0 { + return nil, fmt.Errorf("%w: negative region offset", ErrUnsupportedFormat) + } + n, err := validateAllocSize(size, fileSize) + if err != nil { + return nil, err } - buf := make([]byte, size) + buf := make([]byte, n) if _, err := r.ReadAt(buf, offset); err != nil { return nil, fmt.Errorf("reading region at %d: %w", offset, err) } @@ -126,7 +147,11 @@ func (r *Reader) ReadFile(path string) ([]byte, error) { return nil, fmt.Errorf("unrealpak: %s: entry header size %d disagrees with index size %d", path, size, e.Size) } - buf := make([]byte, e.Size) + n, err := validateAllocSize(e.Size, r.fileSize) + if err != nil { + return nil, fmt.Errorf("unrealpak: %s: %w", path, err) + } + buf := make([]byte, n) if _, err := r.f.ReadAt(buf, e.offset+storedHeaderSize); err != nil { return nil, fmt.Errorf("unrealpak: reading %s: %w", path, err) } @@ -190,7 +215,7 @@ func readFooter(r io.ReaderAt, fileSize int64) (footer, error) { // index (hash -> record offset) and a full directory index // (directory -> file -> record offset). Enumeration uses the directory index, // which is the only one that carries real path strings. -func parseIndex(f io.ReaderAt, index []byte) ([]readerEntry, error) { +func parseIndex(f io.ReaderAt, index []byte, fileSize int64) ([]readerEntry, error) { c := &cursor{b: index} c.fstring() // MountPoint: recorded for the engine's benefit, unused here numEntries := c.i32() @@ -216,10 +241,10 @@ func parseIndex(f io.ReaderAt, index []byte) ([]readerEntry, error) { // Verify the path-hash index's hash even though enumeration does not use // it: it is part of the format's integrity chain, and a pak whose // sub-index hashes don't hold is not one to trust. - if _, err := readRegion(f, pathHash.offset, pathHash.size, pathHash.hash); err != nil { + if _, err := readRegion(f, pathHash.offset, pathHash.size, fileSize, pathHash.hash); err != nil { return nil, fmt.Errorf("path hash index: %w", err) } - dirBuf, err := readRegion(f, fullDir.offset, fullDir.size, fullDir.hash) + dirBuf, err := readRegion(f, fullDir.offset, fullDir.size, fileSize, fullDir.hash) if err != nil { return nil, fmt.Errorf("full directory index: %w", err) } @@ -351,7 +376,10 @@ type cursor struct { func (c *cursor) take(n int) []byte { if c.err != nil { - return make([]byte, n) + // A corrupted length field can make n negative even on this + // already-failed path; make([]byte, negative) panics, so clamp here + // too rather than only on the fresh-error branch below. + return make([]byte, max(n, 0)) } if n < 0 || c.pos+n > len(c.b) { c.err = io.ErrUnexpectedEOF diff --git a/internal/unrealpak/reader_test.go b/internal/unrealpak/reader_test.go index c6c17e6..58e4678 100644 --- a/internal/unrealpak/reader_test.go +++ b/internal/unrealpak/reader_test.go @@ -229,3 +229,120 @@ func TestReader_ReadFile_RejectsCompressedEntry(t *testing.T) { t.Fatalf("ReadFile error = %v, want ErrUnsupportedFormat", err) } } + +func TestValidateAllocSize(t *testing.T) { + tests := []struct { + name string + size int64 + fileSize int64 + wantErr bool + want int + }{ + {"valid, well within file", 10, 1000, false, 10}, + {"valid, exactly file size", 1000, 1000, false, 1000}, + {"negative (e.g. a 64-bit field with its top bit set, cast to int64)", -1, 1000, true, 0}, + {"exceeds file size", 1001, 1000, true, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := validateAllocSize(tt.size, tt.fileSize) + if tt.wantErr { + if err == nil { + t.Fatalf("validateAllocSize(%d, %d) = %d, nil; want error", tt.size, tt.fileSize, got) + } + if !errors.Is(err, ErrUnsupportedFormat) { + t.Errorf("error = %v, want ErrUnsupportedFormat", err) + } + return + } + if err != nil { + t.Fatalf("validateAllocSize(%d, %d): %v", tt.size, tt.fileSize, err) + } + if got != tt.want { + t.Errorf("validateAllocSize(%d, %d) = %d, want %d", tt.size, tt.fileSize, got, tt.want) + } + }) + } +} + +// readRegion must reject an invalid size (or offset) before ever attempting +// to read — a corrupted size field must not drive an unbounded allocation or +// even a spurious I/O attempt. panicReaderAt fails the test if readRegion +// reaches the ReadAt call at all. +type readerAtFunc func([]byte, int64) (int, error) + +func (f readerAtFunc) ReadAt(p []byte, off int64) (int, error) { return f(p, off) } + +func TestReadRegion_RejectsInvalidSizeOrOffsetBeforeReading(t *testing.T) { + panicReader := readerAtFunc(func(p []byte, off int64) (int, error) { + panic("readRegion must not read when offset/size is invalid") + }) + var want [20]byte + + if _, err := readRegion(panicReader, 0, -1, 1000, want); !errors.Is(err, ErrUnsupportedFormat) { + t.Errorf("negative size: error = %v, want ErrUnsupportedFormat", err) + } + if _, err := readRegion(panicReader, 0, 2000, 1000, want); !errors.Is(err, ErrUnsupportedFormat) { + t.Errorf("oversized size: error = %v, want ErrUnsupportedFormat", err) + } + if _, err := readRegion(panicReader, -1, 10, 1000, want); !errors.Is(err, ErrUnsupportedFormat) { + t.Errorf("negative offset: error = %v, want ErrUnsupportedFormat", err) + } +} + +// ReadFile must reject a corrupted entry-size field before allocating the +// payload buffer, whether the corruption makes the field negative (a 64-bit +// UncompressedSize with its top bit set, cast to int64) or merely larger +// than the file that supposedly contains it. Constructed directly against a +// Reader/readerEntry rather than through a full on-disk fixture: the +// interesting case is an internally-consistent header+index pair that still +// disagrees with reality, not a hash-gated corruption (which a different, +// already-tested path already catches). +func TestReader_ReadFile_RejectsInvalidSizeField(t *testing.T) { + tests := []struct { + name string + size int64 + fileSize int64 + }{ + {"negative size", -1, 1000}, + {"size exceeding the file it's claimed to live in", 1 << 40, 1000}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hdr := storedEntryHeader(tt.size, []byte("irrelevant")) // payload is never read: the guard fires first + pakPath := filepath.Join(t.TempDir(), "test.pak") + if err := os.WriteFile(pakPath, hdr, 0o644); err != nil { + t.Fatal(err) + } + f, err := os.Open(pakPath) + if err != nil { + t.Fatal(err) + } + defer f.Close() //nolint:errcheck + + r := &Reader{ + f: f, + fileSize: tt.fileSize, + entries: []readerEntry{ + {FileEntry: FileEntry{Path: "x.json", Size: tt.size}, offset: 0, method: 0}, + }, + } + + if _, err := r.ReadFile("x.json"); !errors.Is(err, ErrUnsupportedFormat) { + t.Fatalf("ReadFile error = %v, want ErrUnsupportedFormat", err) + } + }) + } +} + +// A cursor that has already latched an error must not panic when asked to +// take a negative length (reachable via a corrupted length field parsed +// earlier in the same structure, e.g. fstring's negative-length check +// latches c.err and later cursor calls in the same parse can still run). +func TestCursor_Take_ClampsNegativeLengthOnErrLatchedPath(t *testing.T) { + c := &cursor{b: []byte{1, 2, 3}, err: errors.New("boom")} + got := c.take(-5) // must not panic + if len(got) != 0 { + t.Errorf("take(-5) on an err-latched cursor = %v, want empty", got) + } +} From 8b282f60b967b242f666b3ed35fe90c46e796840 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Fri, 31 Jul 2026 21:59:44 -0400 Subject: [PATCH 20/96] fix: guard writer index overflow, cap dump tar reads, fix filename fallback (review) (#136) PR #171 Copilot round-2 review fixes: - unrealpak: Writer.Close now rejects, via a new checkEncodedLocationFits helper, an encoded-index length exceeding math.MaxInt32 before storing it as an int32 location -- previously a >2 GiB index would silently wrap/go-negative there, corrupting every location recorded from that point on. Consistent with the existing >4GiB offset/size error style. - icarus: fetchTree now rejects a tar entry whose declared header size exceeds a 64 MiB per-table cap (the largest real table is 7.3 MB) before reading it, and reads the body through io.LimitReader bounded by the declared size -- a network-fetched, third-party archive's size field is not trusted for allocation. - icarus: fileNameFromURL's fallback now returns a dotted name ('mod.') instead of a bare extension ('exmodz'/'pak'), which silently defeated isExmodzFile's '.exmodz' suffix check and compiledFileName's filepath.Ext-based rename. A parsed basename that exists but has no extension of its own gets the expected extension appended rather than being discarded. Also guards a '..' basename into the fallback path (closes the final review's fileNameFromURL '..' minor). New tests: TestCheckEncodedLocationFits, TestDumpStore_DumpForBuild_RejectsOversizedTarEntry, TestFileNameFromURL (table-driven). --- internal/source/icarus/datadump.go | 12 ++++++- internal/source/icarus/datadump_test.go | 42 +++++++++++++++++++++++++ internal/source/icarus/icarus.go | 19 +++++++++-- internal/source/icarus/icarus_test.go | 28 +++++++++++++++++ internal/unrealpak/writer.go | 18 +++++++++++ internal/unrealpak/writer_test.go | 16 ++++++++++ 6 files changed, 131 insertions(+), 4 deletions(-) diff --git a/internal/source/icarus/datadump.go b/internal/source/icarus/datadump.go index 7fd8a1e..5c2d30d 100644 --- a/internal/source/icarus/datadump.go +++ b/internal/source/icarus/datadump.go @@ -31,6 +31,12 @@ const defaultDumpTreeURL = "https://codeload.github.com/GODOFMINECRAFT4/IcarusDa // grow while refusing to stream an unbounded body into memory. const maxDumpBytes = 256 << 20 +// maxTarEntrySize caps a single table's decompressed size. The largest real +// table (Items/D_ItemsStatic.json, per the base pak) is 7.3 MB; 64 MiB leaves +// generous headroom while refusing to trust an unbounded or lying size field +// in a tar header from a third-party, network-fetched archive. +const maxTarEntrySize = 64 << 20 + // Build identifies the installed game, read from Icarus/Config/version.json. // Note this carries no week number — nothing in the install does. Week // agreement is established by content comparison, not by this value. @@ -216,7 +222,11 @@ func (s *DumpStore) fetchTree(ctx context.Context, url string) (*Dump, error) { if rel == "" || strings.HasPrefix(rel, "data/") { continue } - body, err := io.ReadAll(tr) + if hdr.Size > maxTarEntrySize { + return nil, fmt.Errorf("icarus: base-table dump entry %s declares a %d-byte size, "+ + "exceeding the %d-byte per-table cap", rel, hdr.Size, maxTarEntrySize) + } + body, err := io.ReadAll(io.LimitReader(tr, hdr.Size)) if err != nil { return nil, fmt.Errorf("icarus: reading %s from base-table dump: %w", rel, err) } diff --git a/internal/source/icarus/datadump_test.go b/internal/source/icarus/datadump_test.go index ba7b9a8..6681730 100644 --- a/internal/source/icarus/datadump_test.go +++ b/internal/source/icarus/datadump_test.go @@ -295,6 +295,48 @@ func TestDumpStore_DumpForBuild_NetworkFailure_IsActionable(t *testing.T) { } } +// A tar entry whose declared header size exceeds the per-table cap must be +// rejected before any content is read — guards against a network-fetched, +// third-party archive with a corrupt or lying size field driving an +// unbounded allocation. The fixture never writes real content matching the +// declared size (impractical at 64+ MiB): tw.WriteHeader alone already +// produces a header a tar.Reader can parse, and the cap fires on hdr.Size +// alone, before fetchTree ever attempts to read the entry's body. +func TestDumpStore_DumpForBuild_RejectsOversizedTarEntry(t *testing.T) { + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + hdr := &tar.Header{ + Name: "IcarusData-test/Huge/D_Huge.json", + Mode: 0o644, + Size: maxTarEntrySize + 1, + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + // Deliberately not calling tw.Close() or writing the declared body. + if err := gz.Close(); err != nil { + t.Fatal(err) + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(buf.Bytes()) + })) + defer srv.Close() + + store := newDumpStore(t.TempDir(), srv.Client()) + store.treeURL = srv.URL + + pak := writeTestBasePak(t, map[string][]byte{"a/B.json": []byte("{}")}) + _, err := store.DumpForBuild(context.Background(), pak, "") + if err == nil { + t.Fatal("expected an error for an oversized tar entry, got nil") + } + if !strings.Contains(err.Error(), "Huge/D_Huge.json") { + t.Errorf("error %q should name the offending entry", err) + } +} + // A base pak entry that fails to read for a reason OTHER than // unrealpak.ErrUnsupportedFormat (corruption, a truncated payload, an I/O // error) must fail validateDump loudly, not be silently folded into the diff --git a/internal/source/icarus/icarus.go b/internal/source/icarus/icarus.go index 4d63a4d..4d793a8 100644 --- a/internal/source/icarus/icarus.go +++ b/internal/source/icarus/icarus.go @@ -223,14 +223,27 @@ func mapDoc(d firestoreDoc) domain.Mod { } } +// fileNameFromURL derives a download's file name from its URL, falling back +// to a synthesized "mod." name (never a bare, dot-less +// fallbackExt) when the URL yields nothing usable. A dot-less fallback would +// silently defeat both isExmodzFile's case-insensitive ".exmodz" suffix +// check and compiledFileName's filepath.Ext-based rename in Service — a +// downloaded file named e.g. "exmodz" would never route through Compile. +// A parsed basename that exists but carries no extension of its own gets +// fallbackExt appended rather than being discarded outright, preserving +// whatever real name the URL offered. func fileNameFromURL(rawURL, fallbackExt string) string { + fallback := "mod." + fallbackExt u, err := url.Parse(rawURL) if err != nil || u.Path == "" { - return fallbackExt + return fallback } base := path.Base(u.Path) - if base == "." || base == "/" { - return fallbackExt + if base == "." || base == "/" || base == ".." { + return fallback + } + if path.Ext(base) == "" { + return base + "." + fallbackExt } return base } diff --git a/internal/source/icarus/icarus_test.go b/internal/source/icarus/icarus_test.go index f21b877..2fe4414 100644 --- a/internal/source/icarus/icarus_test.go +++ b/internal/source/icarus/icarus_test.go @@ -85,6 +85,34 @@ func TestIcarus_GetModFiles_ReturnsExmodzAndPak(t *testing.T) { } } +// The fallback must always be a dotted name (never a bare "exmodz"/"pak"), +// or isExmodzFile's ".exmodz" suffix check and compiledFileName's +// filepath.Ext-based rename in Service would both silently misroute the +// download instead of failing loudly (#136 review round 2). +func TestFileNameFromURL(t *testing.T) { + tests := []struct { + name string + rawURL string + fallbackExt string + want string + }{ + {"basename with extension is used as-is", "https://x/mods/Bear_Mount.exmodz", "exmodz", "Bear_Mount.exmodz"}, + {"basename without an extension gets fallbackExt appended", "https://x/mods/Bear_Mount", "exmodz", "Bear_Mount.exmodz"}, + {"root path falls back to a dotted name", "https://x/", "pak", "mod.pak"}, + {"empty path falls back to a dotted name", "https://x", "pak", "mod.pak"}, + {"a path of exactly '..' falls back to a dotted name", "https://x/mods/..", "exmodz", "mod.exmodz"}, + {"an unparseable URL falls back to a dotted name", "://not a url", "pak", "mod.pak"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := fileNameFromURL(tt.rawURL, tt.fallbackExt) + if got != tt.want { + t.Errorf("fileNameFromURL(%q, %q) = %q, want %q", tt.rawURL, tt.fallbackExt, got, tt.want) + } + }) + } +} + // TestIcarus_Compile_WithoutDataDir_FailsLoudly pins that a source // constructed via New but never wired with SetDataDir (e.g. a registration // path that forgets the optional-setter call) fails loudly instead of diff --git a/internal/unrealpak/writer.go b/internal/unrealpak/writer.go index 31f9be6..8489bf9 100644 --- a/internal/unrealpak/writer.go +++ b/internal/unrealpak/writer.go @@ -67,6 +67,20 @@ func (w *Writer) AddFile(mountPath string, data []byte) error { return nil } +// checkEncodedLocationFits validates that encodedLen — the offset the next +// entry's record is about to be stored at within the encoded index blob — +// still fits the int32 location field the format stores in the path-hash and +// full directory indexes. Extracted from Close so the boundary can be tested +// without constructing a >2 GiB fixture: encoded.Len() growing past +// math.MaxInt32 would otherwise silently wrap (even go negative) when cast to +// int32, corrupting every location recorded from that point on. +func checkEncodedLocationFits(encodedLen int) error { + if encodedLen > math.MaxInt32 { + return fmt.Errorf("encoded index exceeds the 32-bit location field this writer emits (%d bytes)", encodedLen) + } + return nil +} + // Close assembles the data section and all three index structures, writes them // with the footer, and closes the file. func (w *Writer) Close() error { @@ -90,6 +104,10 @@ func (w *Writer) Close() error { data.Write(storedEntryHeader(size, file.data)) data.Write(file.data) + if err := checkEncodedLocationFits(encoded.Len()); err != nil { + w.f.Close() //nolint:errcheck + return fmt.Errorf("unrealpak: %s: %w", file.path, err) + } locations[file.path] = int32(encoded.Len()) // The 12-byte stored record: offset/uncompressed-size/size all // 32-bit-safe, method 0, no compression blocks. diff --git a/internal/unrealpak/writer_test.go b/internal/unrealpak/writer_test.go index 7a7d1a6..6edfbd0 100644 --- a/internal/unrealpak/writer_test.go +++ b/internal/unrealpak/writer_test.go @@ -3,6 +3,7 @@ package unrealpak import ( "bytes" "encoding/binary" + "math" "os" "path/filepath" "testing" @@ -98,3 +99,18 @@ func TestWriter_AddFile_AfterClose_Errors(t *testing.T) { t.Error("expected error adding file after Close, got nil") } } + +// checkEncodedLocationFits is tested directly on the boundary rather than by +// constructing a >2 GiB encoded-index fixture, which would be impractically +// slow and memory-hungry for a unit test. +func TestCheckEncodedLocationFits(t *testing.T) { + if err := checkEncodedLocationFits(0); err != nil { + t.Errorf("checkEncodedLocationFits(0): %v", err) + } + if err := checkEncodedLocationFits(math.MaxInt32); err != nil { + t.Errorf("checkEncodedLocationFits(MaxInt32): %v", err) + } + if err := checkEncodedLocationFits(math.MaxInt32 + 1); err == nil { + t.Error("checkEncodedLocationFits(MaxInt32+1) = nil, want error") + } +} From e18b0bd2d81706a296525fefc95f1d01dd423449 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Fri, 31 Jul 2026 22:10:38 -0400 Subject: [PATCH 21/96] fix: normalize exmodz entry matching, drop dead cacheDir, guard int width (review) (#136) PR #171 Copilot round-3 review fixes (suppressed-comment items): - unrealpak: validateAllocSize also rejects size > math.MaxInt before the caller's int(size) cast -- a no-op on a 64-bit build (where int is 64 bits, same as size's type) but load-bearing on a 32-bit one, where a size that already passed the fileSize check could still overflow int and wrap negative on the cast. - icarus: DumpStore's dead cacheDir field is removed (it was never read -- every call re-fetches over the network) along with newDumpStore's now-pointless cacheDir parameter and SetDataDir's filepath.Join plumbing that built it; DumpStore's doc comment now says what it actually does. SetDataDir keeps its dataDir parameter (required by the shared interface{ SetDataDir(string) } duck-typed contract root.go's registerSource calls uniformly across sources) but no longer uses its value. - icarus: ParseExmodz now normalizes zip entry backslashes to forward slashes and matches the manifest's 'Extracted Mods/' prefix + '.EXMOD' suffix, and an asset's .uasset/.uexp extension, case-insensitively, so a Windows-built .EXMODZ's differently-cased or backslash-separated entries are no longer silently dropped. Multiple candidate manifests is now a loud, named error instead of silently keeping whichever the zip directory listed last. Asset keys are stored under their normalized (forward-slash, original-case) form; compile.go's sanitizeAssetPath still composes correctly against this -- its own backslash normalization becomes a harmless no-op on an already-normalized string, not a double-handling bug (noted in the report). New/extended tests: TestValidateAllocSize (math.MaxInt boundary), TestParseExmodz_NormalizesBackslashNames, TestParseExmodz_MatchesCaseInsensitively, TestParseExmodz_MultipleManifests_Errors. --- internal/source/icarus/compile_test.go | 2 +- internal/source/icarus/datadump.go | 10 ++- internal/source/icarus/datadump_test.go | 16 ++-- internal/source/icarus/exmodz.go | 61 ++++++++++---- internal/source/icarus/exmodz_test.go | 103 ++++++++++++++++++++++++ internal/source/icarus/icarus.go | 23 +++--- internal/unrealpak/reader.go | 17 ++-- internal/unrealpak/reader_test.go | 10 +++ 8 files changed, 197 insertions(+), 45 deletions(-) diff --git a/internal/source/icarus/compile_test.go b/internal/source/icarus/compile_test.go index 51a9560..9fe6503 100644 --- a/internal/source/icarus/compile_test.go +++ b/internal/source/icarus/compile_test.go @@ -27,7 +27,7 @@ func testDumpStore(t *testing.T, files map[string][]byte) *DumpStore { _, _ = w.Write(tarGz(t, "IcarusData-test", entries)) })) t.Cleanup(srv.Close) - store := newDumpStore(t.TempDir(), srv.Client()) + store := newDumpStore(srv.Client()) store.treeURL = srv.URL return store } diff --git a/internal/source/icarus/datadump.go b/internal/source/icarus/datadump.go index 5c2d30d..d1db6ae 100644 --- a/internal/source/icarus/datadump.go +++ b/internal/source/icarus/datadump.go @@ -90,15 +90,17 @@ func (d *Dump) Table(rel string) ([]byte, bool) { return b, ok } -// DumpStore fetches and caches base-table dumps. +// DumpStore fetches base-table dumps (hosted, or from a local directory +// override — see DumpForBuild). It does not cache to disk: every call that +// doesn't supply a localDumpDir re-fetches the tree over the network. Adding +// caching is a real, tracked follow-up, not implemented here (YAGNI). type DumpStore struct { - cacheDir string httpClient *http.Client treeURL string // overridable in tests } -func newDumpStore(cacheDir string, httpClient *http.Client) *DumpStore { - return &DumpStore{cacheDir: cacheDir, httpClient: httpClient, treeURL: defaultDumpTreeURL} +func newDumpStore(httpClient *http.Client) *DumpStore { + return &DumpStore{httpClient: httpClient, treeURL: defaultDumpTreeURL} } // DumpForBuild loads the base data tables and returns them only if they match diff --git a/internal/source/icarus/datadump_test.go b/internal/source/icarus/datadump_test.go index 6681730..dd5f148 100644 --- a/internal/source/icarus/datadump_test.go +++ b/internal/source/icarus/datadump_test.go @@ -148,7 +148,7 @@ func TestDumpStore_DumpForBuild_AcceptsMatchingDump(t *testing.T) { })) defer srv.Close() - store := newDumpStore(t.TempDir(), srv.Client()) + store := newDumpStore(srv.Client()) store.treeURL = srv.URL // test seam dump, err := store.DumpForBuild(context.Background(), pak, "") @@ -174,7 +174,7 @@ func TestDumpStore_DumpForBuild_RejectsWrongWeek(t *testing.T) { })) defer srv.Close() - store := newDumpStore(t.TempDir(), srv.Client()) + store := newDumpStore(srv.Client()) store.treeURL = srv.URL _, err := store.DumpForBuild(context.Background(), pak, "") @@ -215,7 +215,7 @@ func TestDumpStore_DumpForBuild_LocalDirOverridesFetch(t *testing.T) { w.WriteHeader(http.StatusInternalServerError) })) defer srv.Close() - store := newDumpStore(t.TempDir(), srv.Client()) + store := newDumpStore(srv.Client()) store.treeURL = srv.URL dump, err := store.DumpForBuild(context.Background(), pak, local) @@ -239,7 +239,7 @@ func TestDumpStore_DumpForBuild_LocalDirAlreadyCRLF(t *testing.T) { pak := writeTestBasePak(t, map[string][]byte{rel: []byte(shipped)}) local := writeLocalDump(t, map[string]string{rel: shipped}) - store := newDumpStore(t.TempDir(), http.DefaultClient) + store := newDumpStore(http.DefaultClient) store.treeURL = "http://127.0.0.1:0/never-used" if _, err := store.DumpForBuild(context.Background(), pak, local); err != nil { @@ -254,7 +254,7 @@ func TestDumpStore_DumpForBuild_LocalDirWrongWeek_Rejected(t *testing.T) { pak := writeTestBasePak(t, map[string][]byte{rel: []byte("{\r\n \"Rows\": [1]\r\n}")}) local := writeLocalDump(t, map[string]string{rel: "{\n \"Rows\": []\n}"}) - store := newDumpStore(t.TempDir(), http.DefaultClient) + store := newDumpStore(http.DefaultClient) store.treeURL = "http://127.0.0.1:0/never-used" _, err := store.DumpForBuild(context.Background(), pak, local) @@ -271,7 +271,7 @@ func TestDumpStore_DumpForBuild_LocalDirWrongWeek_Rejected(t *testing.T) { func TestDumpStore_DumpForBuild_LocalDirEmpty_IsActionable(t *testing.T) { pak := writeTestBasePak(t, map[string][]byte{"a/B.json": []byte("{}")}) - store := newDumpStore(t.TempDir(), http.DefaultClient) + store := newDumpStore(http.DefaultClient) _, err := store.DumpForBuild(context.Background(), pak, t.TempDir()) if err == nil { @@ -285,7 +285,7 @@ func TestDumpStore_DumpForBuild_NetworkFailure_IsActionable(t *testing.T) { })) defer srv.Close() - store := newDumpStore(t.TempDir(), srv.Client()) + store := newDumpStore(srv.Client()) store.treeURL = srv.URL pak := writeTestBasePak(t, map[string][]byte{"a/B.json": []byte("{}")}) @@ -324,7 +324,7 @@ func TestDumpStore_DumpForBuild_RejectsOversizedTarEntry(t *testing.T) { })) defer srv.Close() - store := newDumpStore(t.TempDir(), srv.Client()) + store := newDumpStore(srv.Client()) store.treeURL = srv.URL pak := writeTestBasePak(t, map[string][]byte{"a/B.json": []byte("{}")}) diff --git a/internal/source/icarus/exmodz.go b/internal/source/icarus/exmodz.go index d0c1a79..bd47ff9 100644 --- a/internal/source/icarus/exmodz.go +++ b/internal/source/icarus/exmodz.go @@ -20,50 +20,77 @@ type ExmodzBundle struct { // bundled assets. The manifest lives at "Extracted Mods/.EXMOD" in // every sample seen so far; this looks for any "*.EXMOD" file under an // "Extracted Mods/" prefix rather than hard-coding the mod name, since that -// varies per mod. +// varies per mod. Matching (both the manifest's prefix/suffix and the asset +// extensions below) is done on the entry name with backslashes normalized to +// forward slashes and case folded — some .EXMODZ producers are Windows tools +// and zip entry casing is not guaranteed — but stored asset keys keep their +// original case, only the slash direction is normalized. func ParseExmodz(zipData []byte) (*ExmodzBundle, error) { zr, err := zip.NewReader(bytes.NewReader(zipData), int64(len(zipData))) if err != nil { return nil, fmt.Errorf("icarus: opening .EXMODZ: %w", err) } - bundle := &ExmodzBundle{Assets: make(map[string][]byte)} - var manifestPath string + var manifests []*zip.File for _, f := range zr.File { - if strings.HasPrefix(f.Name, "Extracted Mods/") && strings.HasSuffix(f.Name, ".EXMOD") { - manifestPath = f.Name - data, err := readZipFile(f) - if err != nil { - return nil, fmt.Errorf("icarus: reading %s: %w", f.Name, err) - } - bundle.Diff, err = ParseExmod(data) - if err != nil { - return nil, err - } - continue + lower := strings.ToLower(normalizeZipName(f.Name)) + if strings.HasPrefix(lower, "extracted mods/") && strings.HasSuffix(lower, ".exmod") { + manifests = append(manifests, f) } } - if manifestPath == "" { + switch len(manifests) { + case 0: return nil, fmt.Errorf("icarus: .EXMODZ has no Extracted Mods/*.EXMOD manifest") + case 1: + // exactly one candidate — proceed below + default: + names := make([]string, len(manifests)) + for i, f := range manifests { + names[i] = f.Name + } + return nil, fmt.Errorf("icarus: .EXMODZ has %d ambiguous Extracted Mods/*.EXMOD manifests: %v", + len(manifests), names) + } + manifestFile := manifests[0] + manifestPath := manifestFile.Name // original (un-normalized) name, used only to exclude this entry below + + manifestData, err := readZipFile(manifestFile) + if err != nil { + return nil, fmt.Errorf("icarus: reading %s: %w", manifestPath, err) } + diff, err := ParseExmod(manifestData) + if err != nil { + return nil, err + } + bundle := &ExmodzBundle{Diff: diff, Assets: make(map[string][]byte)} for _, f := range zr.File { if f.Name == manifestPath || f.FileInfo().IsDir() { continue } - if !strings.HasSuffix(f.Name, ".uasset") && !strings.HasSuffix(f.Name, ".uexp") { + normalized := normalizeZipName(f.Name) + lower := strings.ToLower(normalized) + if !strings.HasSuffix(lower, ".uasset") && !strings.HasSuffix(lower, ".uexp") { continue // skip readme/image/other non-asset files — never placed into the output pak } data, err := readZipFile(f) if err != nil { return nil, fmt.Errorf("icarus: reading asset %s: %w", f.Name, err) } - bundle.Assets[f.Name] = data + bundle.Assets[normalized] = data } return bundle, nil } +// normalizeZipName converts a zip entry's backslashes to forward slashes. +// Zip is a forward-slash format, but some Windows-built .EXMODZ archives +// store entries with backslashes anyway; this makes matching and asset keys +// consistent regardless of which the producing tool used. +func normalizeZipName(name string) string { + return strings.ReplaceAll(name, `\`, "/") +} + func readZipFile(f *zip.File) ([]byte, error) { rc, err := f.Open() if err != nil { diff --git a/internal/source/icarus/exmodz_test.go b/internal/source/icarus/exmodz_test.go index bf73b82..e032e15 100644 --- a/internal/source/icarus/exmodz_test.go +++ b/internal/source/icarus/exmodz_test.go @@ -3,6 +3,7 @@ package icarus import ( "archive/zip" "bytes" + "strings" "testing" ) @@ -58,6 +59,108 @@ func mapKeys(m map[string][]byte) []string { return out } +// Some .EXMODZ producers are Windows tools and store entry names with +// backslashes; ParseExmodz must normalize them for matching and store asset +// keys under the normalized (forward-slash) form (#136 review round 3). +func TestParseExmodz_NormalizesBackslashNames(t *testing.T) { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + + manifest := `{"name":"Bear Mount","Rows":[]}` + w, err := zw.Create(`Extracted Mods\Bear_Mount.EXMOD`) + if err != nil { + t.Fatal(err) + } + w.Write([]byte(manifest)) //nolint:errcheck + + assetW, err := zw.Create(`Bear_Mount\ASS\ITM\SK_ITM_Saddle_Bear.uasset`) + if err != nil { + t.Fatal(err) + } + assetW.Write([]byte("fake-uasset-bytes")) //nolint:errcheck + + if err := zw.Close(); err != nil { + t.Fatal(err) + } + + bundle, err := ParseExmodz(buf.Bytes()) + if err != nil { + t.Fatalf("ParseExmodz: %v", err) + } + if bundle.Diff == nil || bundle.Diff.Name != "Bear Mount" { + t.Fatalf("Diff = %+v", bundle.Diff) + } + asset, ok := bundle.Assets["Bear_Mount/ASS/ITM/SK_ITM_Saddle_Bear.uasset"] + if !ok { + t.Fatalf("Assets missing expected forward-slash key; got keys: %v", mapKeys(bundle.Assets)) + } + if string(asset) != "fake-uasset-bytes" { + t.Errorf("asset content = %q", asset) + } +} + +// The manifest's "Extracted Mods/" prefix + ".EXMOD" suffix, and an asset's +// .uasset/.uexp extension, must match regardless of case — a differently +// cased but otherwise valid entry must not be silently dropped. +func TestParseExmodz_MatchesCaseInsensitively(t *testing.T) { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + + manifest := `{"name":"Bear Mount","Rows":[]}` + w, err := zw.Create("extracted mods/Bear_Mount.exmod") + if err != nil { + t.Fatal(err) + } + w.Write([]byte(manifest)) //nolint:errcheck + + assetW, err := zw.Create("Bear_Mount/ASS/ITM/SK_ITM_Saddle_Bear.UASSET") + if err != nil { + t.Fatal(err) + } + assetW.Write([]byte("fake-uasset-bytes")) //nolint:errcheck + + if err := zw.Close(); err != nil { + t.Fatal(err) + } + + bundle, err := ParseExmodz(buf.Bytes()) + if err != nil { + t.Fatalf("ParseExmodz: %v", err) + } + if bundle.Diff == nil || bundle.Diff.Name != "Bear Mount" { + t.Fatalf("Diff = %+v", bundle.Diff) + } + if _, ok := bundle.Assets["Bear_Mount/ASS/ITM/SK_ITM_Saddle_Bear.UASSET"]; !ok { + t.Fatalf("Assets missing case-varying key (original case preserved); got keys: %v", mapKeys(bundle.Assets)) + } +} + +// Two candidate manifests is ambiguous and must fail loudly, naming both — +// not silently pick whichever the zip directory happened to list last. +func TestParseExmodz_MultipleManifests_Errors(t *testing.T) { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + + for _, name := range []string{"Extracted Mods/A.EXMOD", "Extracted Mods/B.EXMOD"} { + w, err := zw.Create(name) + if err != nil { + t.Fatal(err) + } + w.Write([]byte(`{"name":"X","Rows":[]}`)) //nolint:errcheck + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + + _, err := ParseExmodz(buf.Bytes()) + if err == nil { + t.Fatal("expected an error for multiple candidate manifests, got nil") + } + if !strings.Contains(err.Error(), "A.EXMOD") || !strings.Contains(err.Error(), "B.EXMOD") { + t.Errorf("error %q should name both ambiguous manifests", err) + } +} + func TestParseExmodz_NoManifest_Errors(t *testing.T) { var buf bytes.Buffer zw := zip.NewWriter(&buf) diff --git a/internal/source/icarus/icarus.go b/internal/source/icarus/icarus.go index 4d793a8..62e049d 100644 --- a/internal/source/icarus/icarus.go +++ b/internal/source/icarus/icarus.go @@ -6,7 +6,6 @@ import ( "net/http" "net/url" "path" - "path/filepath" "strings" "github.com/DonovanMods/linux-mod-manager/internal/domain" @@ -31,15 +30,21 @@ func New(httpClient *http.Client, projectID string) *Icarus { return &Icarus{firestore: newFirestoreClient(projectID, httpClient)} } -// SetDataDir wires the base-table dump store's cache directory once the -// service's data directory is known. This is a post-construction setter -// rather than a New parameter because Task 8 froze New(httpClient, projectID) -// at exactly those two params — Task 9's call site already depends on that -// signature — so the data dir arrives the same way API keys do: an optional -// setter the registration pipeline calls when present (cmd/lmm/root.go's -// registerSource, mirroring its existing SetAPIKey wiring). +// SetDataDir constructs the base-table dump store once the service's data +// directory is known, gating Compile on it having been called at all (see +// TestIcarus_Compile_WithoutDataDir_FailsLoudly) — DumpStore itself has no +// current use for dataDir's value (it fetches on demand rather than caching +// to disk, see DumpStore's doc comment), so the parameter exists only to +// satisfy the shared `interface{ SetDataDir(string) }` duck-typed contract +// cmd/lmm/root.go's registerSource calls uniformly across sources. This is a +// post-construction setter rather than a New parameter because Task 8 froze +// New(httpClient, projectID) at exactly those two params — Task 9's call +// site already depends on that signature — so the data dir arrives the same +// way API keys do: an optional setter the registration pipeline calls when +// present (mirroring its existing SetAPIKey wiring). func (s *Icarus) SetDataDir(dataDir string) { - s.dumps = newDumpStore(filepath.Join(dataDir, "icarus", "datadump"), s.firestore.httpClient) + _ = dataDir // unused: see doc comment above + s.dumps = newDumpStore(s.firestore.httpClient) } var ( diff --git a/internal/unrealpak/reader.go b/internal/unrealpak/reader.go index a77eceb..c5f1825 100644 --- a/internal/unrealpak/reader.go +++ b/internal/unrealpak/reader.go @@ -6,6 +6,7 @@ import ( "encoding/binary" "fmt" "io" + "math" "os" "sort" "strings" @@ -66,14 +67,18 @@ func Open(path string) (*Reader, error) { } // validateAllocSize checks a length field read from pak data before it is -// used to size a make([]byte, ...) allocation: it must be non-negative and +// used to size a make([]byte, ...) allocation: it must be non-negative, // cannot exceed the pak file's own size — no genuine region or payload can be -// larger than the file that contains it. A field outside that range is -// corruption or a layout this package doesn't understand, never something to -// allocate for (a 64-bit size field with its top bit set becomes negative -// once cast to int64, which is exactly the case this exists to catch). +// larger than the file that contains it — and cannot exceed math.MaxInt, +// since the caller immediately casts the result to int (a no-op check on a +// 64-bit build, where int is 64 bits, but load-bearing on a 32-bit one, +// where a size that passed the fileSize check could still overflow int and +// wrap negative on the cast). A field outside that range is corruption or a +// layout this package doesn't understand, never something to allocate for (a +// 64-bit size field with its top bit set becomes negative once cast to +// int64, which is exactly the case the negative check exists to catch). func validateAllocSize(size, fileSize int64) (int, error) { - if size < 0 || size > fileSize { + if size < 0 || size > fileSize || size > math.MaxInt { return 0, fmt.Errorf("%w: size field %d is invalid for a %d-byte pak", ErrUnsupportedFormat, size, fileSize) } return int(size), nil diff --git a/internal/unrealpak/reader_test.go b/internal/unrealpak/reader_test.go index 58e4678..0112b43 100644 --- a/internal/unrealpak/reader_test.go +++ b/internal/unrealpak/reader_test.go @@ -5,6 +5,7 @@ import ( "crypto/sha1" //nolint:gosec // pak format uses SHA1, not our choice "encoding/binary" "errors" + "math" "os" "path/filepath" "strings" @@ -242,6 +243,15 @@ func TestValidateAllocSize(t *testing.T) { {"valid, exactly file size", 1000, 1000, false, 1000}, {"negative (e.g. a 64-bit field with its top bit set, cast to int64)", -1, 1000, true, 0}, {"exceeds file size", 1001, 1000, true, 0}, + // math.MaxInt itself must still be accepted (inclusive boundary, not + // an off-by-one) -- guards the int(size) cast that follows against + // overflow on a 32-bit build, where int is narrower than int64. On a + // 64-bit build (this test's normal target) math.MaxInt == MaxInt64, + // so size can never actually exceed it: size is itself int64, and + // there is no larger int64 value to construct a "just past the + // boundary" case with. The check is a genuine no-op here and only + // load-bearing on 32-bit -- see validateAllocSize's doc comment. + {"exactly math.MaxInt is still valid (boundary, not exceeded)", math.MaxInt, math.MaxInt, false, math.MaxInt}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 1c261fe9f53ebf3508ebd65bb7c70c336779d512 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Fri, 31 Jul 2026 22:18:30 -0400 Subject: [PATCH 22/96] fix: sort Icarus search results before paginating (review) (#136) PR #171 Copilot round-4 review fix: Search() sliced filtered mods for pagination straight off Firestore's listCollection order, which is not guaranteed stable across runs -- the same page could return different mods between requests. Sort deterministically (Name, then ID as a tiebreak for same-named mods) before slicing, matching the custom api/manifest/directory sources' name-based ordering convention (internal/source/custom/search.go). Extended TestIcarus_Search_FiltersClientSide with a deliberately non-alphabetical mock response order and an empty-query search asserting the returned order is alphabetical by Name -- verified this fails without the fix (stashed icarus.go, reran) and passes with it. --- internal/source/icarus/icarus.go | 14 ++++++++++++ internal/source/icarus/icarus_test.go | 31 +++++++++++++++++++++++---- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/internal/source/icarus/icarus.go b/internal/source/icarus/icarus.go index 62e049d..7102d91 100644 --- a/internal/source/icarus/icarus.go +++ b/internal/source/icarus/icarus.go @@ -6,6 +6,7 @@ import ( "net/http" "net/url" "path" + "sort" "strings" "github.com/DonovanMods/linux-mod-manager/internal/domain" @@ -106,6 +107,19 @@ func (s *Icarus) Search(ctx context.Context, query source.SearchQuery) (source.S } } + // Firestore's listCollection order is not guaranteed stable across runs, + // so the same page could otherwise return different mods on different + // requests. Sort deterministically before slicing, matching the custom + // api/manifest/directory sources' name-based ordering convention + // (internal/source/custom/search.go), with an ID tiebreak for the rare + // case of two mods sharing a name. + sort.SliceStable(mods, func(i, j int) bool { + if mods[i].Name != mods[j].Name { + return mods[i].Name < mods[j].Name + } + return mods[i].ID < mods[j].ID + }) + pageSize := query.PageSize if pageSize <= 0 { pageSize = 20 diff --git a/internal/source/icarus/icarus_test.go b/internal/source/icarus/icarus_test.go index 2fe4414..abf45bd 100644 --- a/internal/source/icarus/icarus_test.go +++ b/internal/source/icarus/icarus_test.go @@ -25,18 +25,27 @@ func modsListHandler(mods []map[string]any) http.HandlerFunc { } } +// The mock server deliberately returns documents in non-alphabetical order +// (Wolf, Bear, Aardvark) — Firestore's listCollection order isn't guaranteed +// stable across runs, so Search must sort before paginating rather than +// trusting (or accidentally reproducing) whatever order the server used. func TestIcarus_Search_FiltersClientSide(t *testing.T) { srv := httptest.NewServer(modsListHandler([]map[string]any{ + {"id": "def", "fields": map[string]any{ + "name": map[string]any{"stringValue": "Wolf Pack"}, "author": map[string]any{"stringValue": "Someone"}, + "description": map[string]any{"stringValue": "Tame wolves"}, "version": map[string]any{"stringValue": "1.0"}, + "files": map[string]any{"mapValue": map[string]any{"fields": map[string]any{"pak": map[string]any{"stringValue": "https://x/wolf.pak"}}}}, + }}, {"id": "abc", "fields": map[string]any{ "name": map[string]any{"stringValue": "Bear Mount"}, "author": map[string]any{"stringValue": "Jimk72"}, "description": map[string]any{"stringValue": "Ride a bear"}, "version": map[string]any{"stringValue": "3.3"}, "compatibility": map[string]any{"stringValue": "w57"}, "files": map[string]any{"mapValue": map[string]any{"fields": map[string]any{"exmodz": map[string]any{"stringValue": "https://x/bear.exmodz"}}}}, }}, - {"id": "def", "fields": map[string]any{ - "name": map[string]any{"stringValue": "Wolf Pack"}, "author": map[string]any{"stringValue": "Someone"}, - "description": map[string]any{"stringValue": "Tame wolves"}, "version": map[string]any{"stringValue": "1.0"}, - "files": map[string]any{"mapValue": map[string]any{"fields": map[string]any{"pak": map[string]any{"stringValue": "https://x/wolf.pak"}}}}, + {"id": "ghi", "fields": map[string]any{ + "name": map[string]any{"stringValue": "Aardvark Delight"}, "author": map[string]any{"stringValue": "Someone"}, + "description": map[string]any{"stringValue": "Burrowing companion"}, "version": map[string]any{"stringValue": "1.0"}, + "files": map[string]any{"mapValue": map[string]any{"fields": map[string]any{"pak": map[string]any{"stringValue": "https://x/aardvark.pak"}}}}, }}, })) defer srv.Close() @@ -54,6 +63,20 @@ func TestIcarus_Search_FiltersClientSide(t *testing.T) { if result.Mods[0].GameID != "icarus" { t.Errorf("GameID = %q, want icarus", result.Mods[0].GameID) } + + all, err := src.Search(context.Background(), source.SearchQuery{Query: ""}) + if err != nil { + t.Fatalf("Search: %v", err) + } + wantOrder := []string{"Aardvark Delight", "Bear Mount", "Wolf Pack"} + if len(all.Mods) != len(wantOrder) { + t.Fatalf("Search(\"\") returned %d mods, want %d", len(all.Mods), len(wantOrder)) + } + for i, want := range wantOrder { + if all.Mods[i].Name != want { + t.Errorf("Search(\"\") order[%d] = %q, want %q (deterministic, alphabetical by Name)", i, all.Mods[i].Name, want) + } + } } func TestIcarus_GetModFiles_ReturnsExmodzAndPak(t *testing.T) { From e03497ce3125c8cf306e0cd44da8ba5a8f771515 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Fri, 31 Jul 2026 22:24:44 -0400 Subject: [PATCH 23/96] docs: correct SetDataDir comment after cacheDir removal (#136) registerSource's comment still said SetDataDir was needed because Icarus's Compile 'needs a cache directory for the base-table dump store' -- stale since round 3 (e18b0bd) removed DumpStore's cacheDir entirely (it was never read; the store fetches on demand). Reworded to describe what SetDataDir actually does today: gate Compile on having been called at all (it constructs the dump store then), with dataDir's value itself unused -- SetDataDir exists only to satisfy the shared interface{ SetDataDir(string) } contract. Comment-only change. --- cmd/lmm/root.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/lmm/root.go b/cmd/lmm/root.go index 6d831f4..b42d48b 100644 --- a/cmd/lmm/root.go +++ b/cmd/lmm/root.go @@ -242,8 +242,11 @@ func registerSources(svc *core.Service, cfgDir, dataDir string) { // wins, warning on customSourceWarnWriter) → API-key resolution (env var via // envKeyFor, falling back to the stored DB token) → SetAPIKey when the // source accepts one → SetDataDir when the source accepts one (Icarus's -// Compile needs a cache directory for the base-table dump store, #136 Task -// 13 — New itself can't take it since Task 8/9 froze its 2-arg signature) → +// Compile is gated on SetDataDir having been called at all: that call +// constructs the base-table dump store. dataDir's value itself is currently +// unused there — that store fetches on demand rather than caching to disk, +// #136 review round 3 — SetDataDir just fulfils the shared interface. New +// itself can't take dataDir since Task 8/9 froze its 2-arg signature) → // RegisterSource. func registerSource(svc *core.Service, src source.ModSource, dataDir string) { id := src.ID() From d562d33aaea3cc15ea90a12c1de947a1979cd922 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 10:16:42 -0400 Subject: [PATCH 24/96] feat: parse the pak footer's CompressionMethods table (#175) --- internal/unrealpak/pak.go | 24 +++++ internal/unrealpak/reader.go | 35 +++++-- internal/unrealpak/zlib_test.go | 173 ++++++++++++++++++++++++++++++++ 3 files changed, 226 insertions(+), 6 deletions(-) create mode 100644 internal/unrealpak/zlib_test.go diff --git a/internal/unrealpak/pak.go b/internal/unrealpak/pak.go index 4c4b76d..61f8f70 100644 --- a/internal/unrealpak/pak.go +++ b/internal/unrealpak/pak.go @@ -45,6 +45,30 @@ const writeVersion int32 = 11 // those and refuses to read their payloads. const storedHeaderSize = 53 +// The footer's CompressionMethods table: 5 fixed-width, NUL-padded name slots +// starting at byte 61. An entry's CompressionMethodIndex is 1-based into it +// (0 means "stored", naming no slot). +const ( + maxCompressionMethods = 5 + compressionMethodNameSize = 32 + compressionMethodsOffset = 61 +) + +// zlibMethodName is the CompressionMethods entry this package can decompress. +// Matched case-insensitively: the name is free-form text written by whatever +// cooked the pak. +const zlibMethodName = "Zlib" + +// maxUncompressedEntrySize caps a single entry's decompressed size. +// +// This deliberately does NOT reuse validateAllocSize's "cannot exceed the pak +// file's own size" rule, which holds for on-disk regions but is simply false +// for decompressed output: Icarus's Items/D_ItemsStatic.json expands to +// 7,304,687 bytes inside a 2,458,743-byte pak. A fixed ceiling is the right +// shape of bound here — it stops a malicious or corrupt UncompressedSize from +// driving an unbounded allocation without rejecting legitimate compression. +const maxUncompressedEntrySize = 512 << 20 + // FileEntry describes one file inside a pak, as returned by Reader.Files. type FileEntry struct { Path string // Mount-relative path, e.g. "Icarus/Content/Data/AI-D_AIGrowth.json" diff --git a/internal/unrealpak/reader.go b/internal/unrealpak/reader.go index c5f1825..126a804 100644 --- a/internal/unrealpak/reader.go +++ b/internal/unrealpak/reader.go @@ -12,11 +12,14 @@ import ( "strings" ) -// Reader provides read access to an uncompressed, unencrypted UE4-range pak. +// Reader provides read access to an unencrypted UE4-range pak. Stored entries +// and Zlib-compressed entries are readable; any other compression method is a +// loud ErrUnsupportedFormat. type Reader struct { f *os.File entries []readerEntry - fileSize int64 // total size of the underlying file, for validateAllocSize + fileSize int64 // total size of the underlying file, for validateAllocSize + methods [maxCompressionMethods]string // this pak's own CompressionMethods table } type readerEntry struct { @@ -63,7 +66,18 @@ func Open(path string) (*Reader, error) { return nil, fmt.Errorf("unrealpak: %s: parsing index: %w", path, err) } - return &Reader{f: f, entries: entries, fileSize: fileSize}, nil + return &Reader{f: f, entries: entries, fileSize: fileSize, methods: ft.methods}, nil +} + +// methodName resolves a 1-based CompressionMethodIndex against this pak's own +// footer table. An index with no corresponding name yields "", which no +// supported method matches, so it falls through to the unsupported-format +// error rather than being silently treated as stored. +func (r *Reader) methodName(method int32) string { + if method < 1 || int(method) > len(r.methods) { + return "" + } + return r.methods[method-1] } // validateAllocSize checks a length field read from pak data before it is @@ -174,6 +188,7 @@ type footer struct { indexSize int64 indexHash [20]byte encryptedIndex bool + methods [maxCompressionMethods]string } // readFooter parses the single 221-byte footer shape this package supports. @@ -206,9 +221,17 @@ func readFooter(r io.ReaderAt, fileSize int64) (footer, error) { return footer{}, fmt.Errorf("%w: pak version %d (this package requires >= %d)", ErrUnsupportedFormat, ft.version, minVersion) } - // The trailing CompressionMethods name table is intentionally left - // unparsed: entries carry a method *index*, and this package only ever - // reads payloads whose index is 0 (stored), which needs no name. + // The trailing CompressionMethods table names each compression method this + // pak uses; entries reference them by 1-based index. It MUST be read from + // this pak's own footer rather than assumed: Icarus's data.pak declares + // ["Zlib"] (so index 1 means Zlib) while its pakchunks declare + // ["Oodle","Zlib"] (so index 1 means Oodle). Assuming one pak's table + // applies to another is exactly the mislabel that sent #136 chasing an + // Oodle blocker that data.pak never had. + for i := range ft.methods { + slot := buf[compressionMethodsOffset+i*compressionMethodNameSize : compressionMethodsOffset+(i+1)*compressionMethodNameSize] + ft.methods[i] = string(bytes.TrimRight(slot, "\x00")) + } return ft, nil } diff --git a/internal/unrealpak/zlib_test.go b/internal/unrealpak/zlib_test.go new file mode 100644 index 0000000..2421d04 --- /dev/null +++ b/internal/unrealpak/zlib_test.go @@ -0,0 +1,173 @@ +package unrealpak + +import ( + "bytes" + "compress/zlib" + "crypto/sha1" //nolint:gosec // pak format uses SHA1, not our choice + "encoding/binary" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +// zlibFixture describes one compressed entry to place in a synthetic pak. +type zlibFixture struct { + path string + blocks [][]byte // each block's PLAINTEXT; each is deflated independently + method int32 // 1-based index into the methods table below +} + +// writeMethodPak hand-builds a version-11 pak whose footer declares methods and +// which holds one compressed entry per fixture. +// +// The Writer only ever emits stored entries, so a compressed fixture has to be +// assembled here. The layout mirrors what a real cooked pak contains: a +// per-entry header carrying the block table, then the deflated blocks, then the +// three index structures and the 221-byte footer. +func writeMethodPak(t *testing.T, methods []string, fixtures []zlibFixture) string { + t.Helper() + const seed uint64 = 0x0123456789ABCDEF + + var data bytes.Buffer + var encoded bytes.Buffer + locations := make(map[string]int32, len(fixtures)) + + for _, fx := range fixtures { + var payload bytes.Buffer + type span struct{ start, end int64 } + hdrSize := compressedHeaderSize(len(fx.blocks)) + var spans []span + var uncompressed int64 + for _, plain := range fx.blocks { + var zbuf bytes.Buffer + zw := zlib.NewWriter(&zbuf) + if _, err := zw.Write(plain); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + start := hdrSize + int64(payload.Len()) + payload.Write(zbuf.Bytes()) + spans = append(spans, span{start, hdrSize + int64(payload.Len())}) + uncompressed += int64(len(plain)) + } + size := int64(payload.Len()) + sum := sha1.Sum(payload.Bytes()) //nolint:gosec + + entryOffset := int64(data.Len()) + var hdr bytes.Buffer + binary.Write(&hdr, binary.LittleEndian, int64(0)) //nolint:errcheck // Offset + binary.Write(&hdr, binary.LittleEndian, size) //nolint:errcheck + binary.Write(&hdr, binary.LittleEndian, uncompressed) //nolint:errcheck + binary.Write(&hdr, binary.LittleEndian, fx.method) //nolint:errcheck + hdr.Write(sum[:]) + binary.Write(&hdr, binary.LittleEndian, int32(len(fx.blocks))) //nolint:errcheck + for _, sp := range spans { + binary.Write(&hdr, binary.LittleEndian, sp.start) //nolint:errcheck + binary.Write(&hdr, binary.LittleEndian, sp.end) //nolint:errcheck + } + hdr.WriteByte(0) // Flags + binary.Write(&hdr, binary.LittleEndian, uint32(65536)) //nolint:errcheck // CompressionBlockSize + if int64(hdr.Len()) != hdrSize { + t.Fatalf("fixture header is %d bytes, compressedHeaderSize says %d", hdr.Len(), hdrSize) + } + data.Write(hdr.Bytes()) + data.Write(payload.Bytes()) + + locations[fx.path] = int32(encoded.Len()) + flags := uint32(1<<31) | uint32(1<<30) | uint32(1<<29) | + uint32(fx.method)<<23 | uint32(len(fx.blocks))<<6 | uint32(65536>>11) + binary.Write(&encoded, binary.LittleEndian, flags) //nolint:errcheck + binary.Write(&encoded, binary.LittleEndian, uint32(entryOffset)) //nolint:errcheck + binary.Write(&encoded, binary.LittleEndian, uint32(uncompressed)) //nolint:errcheck + binary.Write(&encoded, binary.LittleEndian, uint32(size)) //nolint:errcheck + if len(fx.blocks) > 1 { + for _, sp := range spans { + binary.Write(&encoded, binary.LittleEndian, uint32(sp.end-sp.start)) //nolint:errcheck + } + } + } + + // Full directory index: one directory per fixture path. + var fdi bytes.Buffer + binary.Write(&fdi, binary.LittleEndian, int32(len(fixtures))) //nolint:errcheck + for _, fx := range fixtures { + dir, file := splitMountPath(fx.path) + writeFString(&fdi, dir) + binary.Write(&fdi, binary.LittleEndian, int32(1)) //nolint:errcheck + writeFString(&fdi, file) + binary.Write(&fdi, binary.LittleEndian, locations[fx.path]) //nolint:errcheck + } + // Path-hash index, then an empty pruned directory index. + var phi bytes.Buffer + binary.Write(&phi, binary.LittleEndian, int32(len(fixtures))) //nolint:errcheck + for _, fx := range fixtures { + binary.Write(&phi, binary.LittleEndian, hashPath(fx.path, seed)) //nolint:errcheck + binary.Write(&phi, binary.LittleEndian, locations[fx.path]) //nolint:errcheck + } + binary.Write(&phi, binary.LittleEndian, int32(0)) //nolint:errcheck + + phiHash := sha1.Sum(phi.Bytes()) //nolint:gosec + fdiHash := sha1.Sum(fdi.Bytes()) //nolint:gosec + count := int32(len(fixtures)) + indexOffset := int64(data.Len()) + sizing := buildPrimaryIndex(count, seed, 0, 0, phiHash, 0, 0, fdiHash, encoded.Bytes()) + phiOffset := indexOffset + int64(len(sizing)) + fdiOffset := phiOffset + int64(phi.Len()) + index := buildPrimaryIndex(count, seed, phiOffset, int64(phi.Len()), phiHash, + fdiOffset, int64(fdi.Len()), fdiHash, encoded.Bytes()) + indexHash := sha1.Sum(index) //nolint:gosec + + footer := buildFooter(writeVersion, indexOffset, int64(len(index)), indexHash) + for i, name := range methods { + copy(footer[compressionMethodsOffset+i*compressionMethodNameSize:], name) + } + + var out bytes.Buffer + out.Write(data.Bytes()) + out.Write(index) + out.Write(phi.Bytes()) + out.Write(fdi.Bytes()) + out.Write(footer) + + p := filepath.Join(t.TempDir(), "compressed.pak") + if err := os.WriteFile(p, out.Bytes(), 0o644); err != nil { + t.Fatal(err) + } + return p +} + +// The method table is read from the pak's own footer: the SAME index means +// different things in different paks, so an index naming Oodle must be refused +// even though index 1 means Zlib elsewhere. +func TestReadFile_MethodIndexResolvedAgainstThisPaksTable(t *testing.T) { + body := []byte("{}") + p := writeMethodPak(t, []string{"Oodle", "Zlib"}, []zlibFixture{ + {path: "a/Oodled.json", blocks: [][]byte{body}, method: 1}, + {path: "b/Zlibbed.json", blocks: [][]byte{body}, method: 2}, + }) + + r, err := Open(p) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() //nolint:errcheck + + _, err = r.ReadFile("a/Oodled.json") + if !errors.Is(err, ErrUnsupportedFormat) { + t.Fatalf("Oodle entry: err = %v, want ErrUnsupportedFormat", err) + } + if !strings.Contains(err.Error(), "Oodle") { + t.Errorf("Oodle refusal %q should name the method", err) + } + got, err := r.ReadFile("b/Zlibbed.json") + if err != nil { + t.Fatalf("Zlib entry at index 2: %v", err) + } + if !bytes.Equal(got, body) { + t.Errorf("ReadFile = %q, want %q", got, body) + } +} From 56a5e2d1fb957ef3d99443f2635869d820dabda6 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 10:19:48 -0400 Subject: [PATCH 25/96] feat: decompress Zlib pak entries with the standard library (#175) --- internal/unrealpak/reader.go | 187 ++++++++++++++++++++++++++------ internal/unrealpak/zlib_test.go | 94 ++++++++++++++++ 2 files changed, 247 insertions(+), 34 deletions(-) diff --git a/internal/unrealpak/reader.go b/internal/unrealpak/reader.go index 126a804..8c60462 100644 --- a/internal/unrealpak/reader.go +++ b/internal/unrealpak/reader.go @@ -2,6 +2,7 @@ package unrealpak import ( "bytes" + "compress/zlib" "crypto/sha1" //nolint:gosec // pak format uses SHA1, not our choice "encoding/binary" "fmt" @@ -25,8 +26,11 @@ type Reader struct { type readerEntry struct { FileEntry offset int64 // absolute offset of the entry's on-disk header - method int32 // CompressionMethodIndex; 0 = stored. Non-zero entries are - // enumerated but their payloads cannot be read (see ReadFile, Task 3). + method int32 // CompressionMethodIndex; 0 = stored, else a 1-based index + // into the pak footer's CompressionMethods table. + size int64 // on-disk size: compressed for a compressed entry, and equal + // to FileEntry.Size (the uncompressed size) for a stored one. + blocks int // compression block count; 0 for a stored entry. } // Open parses path's footer and index. It does not read file contents — @@ -136,50 +140,162 @@ func (r *Reader) Files() []FileEntry { // ReadFile returns the bytes of the entry at mount-relative path. // // On-disk entry data is preceded by a full FPakEntry header — 53 bytes for a -// stored entry (Offset, Size, UncompressedSize, CompressionMethodIndex, Hash, -// Flags, CompressionBlockSize) — and the index's offset points at that header, -// not the payload. The header is re-read and cross-checked rather than trusted: -// its method and size must agree with the index, and its Hash must match the -// payload's SHA1. Real paks satisfy all three (verified across a whole install), -// so a disagreement means corruption or a layout this package misread. +// stored entry, plus a block table for a compressed one — and the index's +// offset points at that header, not the payload. The header is re-read and +// cross-checked rather than trusted: its method and sizes must agree with the +// index, and its Hash must match the on-disk payload's SHA1. Real paks satisfy +// all of this (verified across a whole install), so a disagreement means +// corruption or a layout this package misread. func (r *Reader) ReadFile(path string) ([]byte, error) { for _, e := range r.entries { if e.Path != path { continue } - // Compression is refused here rather than at index-parse time so that - // Files() can still enumerate real paks, most of whose entries are - // Oodle-compressed. No caller can obtain wrong bytes either way. - if e.method != 0 { - return nil, fmt.Errorf("unrealpak: %s: %w: compressed entry (method %d)", - path, ErrUnsupportedFormat, e.method) + if e.method == 0 { + return r.readStored(path, e) } - hdr := make([]byte, storedHeaderSize) - if _, err := r.f.ReadAt(hdr, e.offset); err != nil { - return nil, fmt.Errorf("unrealpak: %s: reading entry header: %w", path, err) + name := r.methodName(e.method) + if strings.EqualFold(name, zlibMethodName) { + return r.readZlib(path, e) } - if m := int32(binary.LittleEndian.Uint32(hdr[24:28])); m != 0 { - return nil, fmt.Errorf("unrealpak: %s: %w: compressed entry data (method %d)", - path, ErrUnsupportedFormat, m) - } - if size := int64(binary.LittleEndian.Uint64(hdr[8:16])); size != e.Size { - return nil, fmt.Errorf("unrealpak: %s: entry header size %d disagrees with index size %d", - path, size, e.Size) + // Oodle and anything else this package cannot decode stay a hard + // error. Refusing here rather than at index-parse time keeps Files() + // able to enumerate a pak whose entries we cannot all read. + return nil, fmt.Errorf("unrealpak: %s: %w: compression method %q (index %d)", + path, ErrUnsupportedFormat, name, e.method) + } + return nil, fmt.Errorf("unrealpak: %s: %w", path, os.ErrNotExist) +} + +// readStored reads an uncompressed entry: a 53-byte header then the payload. +func (r *Reader) readStored(path string, e readerEntry) ([]byte, error) { + hdr := make([]byte, storedHeaderSize) + if _, err := r.f.ReadAt(hdr, e.offset); err != nil { + return nil, fmt.Errorf("unrealpak: %s: reading entry header: %w", path, err) + } + if m := int32(binary.LittleEndian.Uint32(hdr[24:28])); m != 0 { + return nil, fmt.Errorf("unrealpak: %s: %w: compressed entry data (method %d)", + path, ErrUnsupportedFormat, m) + } + if size := int64(binary.LittleEndian.Uint64(hdr[8:16])); size != e.Size { + return nil, fmt.Errorf("unrealpak: %s: entry header size %d disagrees with index size %d", + path, size, e.Size) + } + n, err := validateAllocSize(e.Size, r.fileSize) + if err != nil { + return nil, fmt.Errorf("unrealpak: %s: %w", path, err) + } + buf := make([]byte, n) + if _, err := r.f.ReadAt(buf, e.offset+storedHeaderSize); err != nil { + return nil, fmt.Errorf("unrealpak: reading %s: %w", path, err) + } + if sum := sha1.Sum(buf); !bytes.Equal(sum[:], hdr[28:48]) { //nolint:gosec + return nil, fmt.Errorf("unrealpak: %s: content hash mismatch", path) + } + return buf, nil +} + +// compressedHeaderSize is the on-disk size of a compressed entry's FPakEntry +// header: the 53-byte stored shape plus a BlockCount(4) and a 16-byte +// (CompressedStart, CompressedEnd) pair per block, inserted between Hash and +// Flags. +func compressedHeaderSize(blocks int) int64 { + return storedHeaderSize + 4 + 16*int64(blocks) +} + +// readZlib reads and reassembles a Zlib-compressed entry. +// +// The entry's payload is split into independently-deflated blocks. The +// authoritative block table lives in the entry's own on-disk header as +// (CompressedStart, CompressedEnd) pairs measured from the entry offset — the +// index's optional block-size list is omitted for a lone unencrypted block, so +// it cannot be relied on. The blocks tile the payload region contiguously and +// their lengths sum to Size; the header Hash covers those on-disk (compressed) +// bytes, not the decompressed result. +// +// This procedure was validated by reconstructing all 298 tables of the real +// Icarus data.pak — 40 stored plus 258 Zlib — byte-for-byte. +// See docs/plans/icarus-quickbms-spike-findings.md. +func (r *Reader) readZlib(path string, e readerEntry) ([]byte, error) { + if e.blocks <= 0 { + return nil, fmt.Errorf("unrealpak: %s: %w: compressed entry declares %d compression blocks", + path, ErrUnsupportedFormat, e.blocks) + } + hdrSize := compressedHeaderSize(e.blocks) + hn, err := validateAllocSize(hdrSize, r.fileSize) + if err != nil { + return nil, fmt.Errorf("unrealpak: %s: entry header: %w", path, err) + } + hdr := make([]byte, hn) + if _, err := r.f.ReadAt(hdr, e.offset); err != nil { + return nil, fmt.Errorf("unrealpak: %s: reading entry header: %w", path, err) + } + if m := int32(binary.LittleEndian.Uint32(hdr[24:28])); m != e.method { + return nil, fmt.Errorf("unrealpak: %s: entry header method %d disagrees with index method %d", + path, m, e.method) + } + if size := int64(binary.LittleEndian.Uint64(hdr[8:16])); size != e.size { + return nil, fmt.Errorf("unrealpak: %s: entry header size %d disagrees with index size %d", + path, size, e.size) + } + if usize := int64(binary.LittleEndian.Uint64(hdr[16:24])); usize != e.Size { + return nil, fmt.Errorf("unrealpak: %s: entry header uncompressed size %d disagrees with index size %d", + path, usize, e.Size) + } + if nb := int64(int32(binary.LittleEndian.Uint32(hdr[48:52]))); nb != int64(e.blocks) { + return nil, fmt.Errorf("unrealpak: %s: entry header block count %d disagrees with index count %d", + path, nb, e.blocks) + } + + pn, err := validateAllocSize(e.size, r.fileSize) + if err != nil { + return nil, fmt.Errorf("unrealpak: %s: %w", path, err) + } + payload := make([]byte, pn) + if _, err := r.f.ReadAt(payload, e.offset+hdrSize); err != nil { + return nil, fmt.Errorf("unrealpak: reading %s: %w", path, err) + } + if sum := sha1.Sum(payload); !bytes.Equal(sum[:], hdr[28:48]) { //nolint:gosec + return nil, fmt.Errorf("unrealpak: %s: content hash mismatch", path) + } + + if e.Size < 0 || e.Size > maxUncompressedEntrySize { + return nil, fmt.Errorf("unrealpak: %s: %w: uncompressed size %d exceeds the %d-byte cap", + path, ErrUnsupportedFormat, e.Size, int64(maxUncompressedEntrySize)) + } + out := make([]byte, 0, e.Size) + for i := 0; i < e.blocks; i++ { + start := int64(binary.LittleEndian.Uint64(hdr[52+i*16 : 60+i*16])) + end := int64(binary.LittleEndian.Uint64(hdr[60+i*16 : 68+i*16])) + // Block bounds are relative to the entry offset and must land inside + // the payload region that follows the header. + if start < hdrSize || end < start || end > hdrSize+e.size { + return nil, fmt.Errorf("unrealpak: %s: block %d spans [%d,%d), outside the entry's payload", + path, i, start, end) } - n, err := validateAllocSize(e.Size, r.fileSize) + zr, err := zlib.NewReader(bytes.NewReader(payload[start-hdrSize : end-hdrSize])) if err != nil { - return nil, fmt.Errorf("unrealpak: %s: %w", path, err) + return nil, fmt.Errorf("unrealpak: %s: block %d: %w", path, i, err) } - buf := make([]byte, n) - if _, err := r.f.ReadAt(buf, e.offset+storedHeaderSize); err != nil { - return nil, fmt.Errorf("unrealpak: reading %s: %w", path, err) + // Read at most one byte more than the declared size still allows, so a + // lying UncompressedSize cannot drive an unbounded read. + remaining := e.Size - int64(len(out)) + chunk, err := io.ReadAll(io.LimitReader(zr, remaining+1)) + zr.Close() //nolint:errcheck // read-only decompressor + if err != nil { + return nil, fmt.Errorf("unrealpak: %s: decompressing block %d: %w", path, i, err) } - if sum := sha1.Sum(buf); !bytes.Equal(sum[:], hdr[28:48]) { //nolint:gosec - return nil, fmt.Errorf("unrealpak: %s: content hash mismatch", path) + if int64(len(chunk)) > remaining { + return nil, fmt.Errorf("unrealpak: %s: decompressed output exceeds the declared uncompressed size %d", + path, e.Size) } - return buf, nil + out = append(out, chunk...) } - return nil, fmt.Errorf("unrealpak: %s: %w", path, os.ErrNotExist) + if int64(len(out)) != e.Size { + return nil, fmt.Errorf("unrealpak: %s: decompressed %d bytes, header declares %d", + path, len(out), e.Size) + } + return out, nil } type footer struct { @@ -374,8 +490,9 @@ func decodeEntry(b []byte, at int) (readerEntry, error) { } offset := read(flags&(1<<31) != 0) uncompressed := read(flags&(1<<30) != 0) + size := uncompressed // a stored entry does not serialize Size; it equals UncompressedSize if method != 0 { - read(flags&(1<<29) != 0) // Size on disk; unused, we refuse to read these payloads + size = read(flags&(1<<29) != 0) } if blockCount > 0 && (blockCount > 1 || encrypted) { c.bytes(4 * blockCount) @@ -390,6 +507,8 @@ func decodeEntry(b []byte, at int) (readerEntry, error) { FileEntry: FileEntry{Size: uncompressed}, offset: offset, method: method, + size: size, + blocks: blockCount, }, nil } diff --git a/internal/unrealpak/zlib_test.go b/internal/unrealpak/zlib_test.go index 2421d04..620cc11 100644 --- a/internal/unrealpak/zlib_test.go +++ b/internal/unrealpak/zlib_test.go @@ -171,3 +171,97 @@ func TestReadFile_MethodIndexResolvedAgainstThisPaksTable(t *testing.T) { t.Errorf("ReadFile = %q, want %q", got, body) } } + +func TestReadFile_ZlibSingleBlock(t *testing.T) { + body := []byte("{\r\n \"Rows\": [1,2,3]\r\n}") + p := writeMethodPak(t, []string{"Zlib"}, []zlibFixture{ + {path: "Factions/D_Factions.json", blocks: [][]byte{body}, method: 1}, + }) + + r, err := Open(p) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() //nolint:errcheck + + got, err := r.ReadFile("Factions/D_Factions.json") + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if !bytes.Equal(got, body) { + t.Errorf("ReadFile = %q, want %q", got, body) + } + if files := r.Files(); len(files) != 1 || files[0].Size != int64(len(body)) { + t.Errorf("Files() = %+v, want one entry sized %d", files, len(body)) + } +} + +// Multi-block reassembly is the case the block table exists for: the blocks +// must be concatenated in order. +func TestReadFile_ZlibMultiBlock(t *testing.T) { + b1 := bytes.Repeat([]byte("alpha "), 400) + b2 := bytes.Repeat([]byte("beta "), 400) + b3 := []byte("tail") + want := append(append(append([]byte{}, b1...), b2...), b3...) + p := writeMethodPak(t, []string{"Zlib"}, []zlibFixture{ + {path: "Items/D_ItemsStatic.json", blocks: [][]byte{b1, b2, b3}, method: 1}, + }) + + r, err := Open(p) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() //nolint:errcheck + + got, err := r.ReadFile("Items/D_ItemsStatic.json") + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if !bytes.Equal(got, want) { + t.Errorf("ReadFile returned %d bytes, want %d (block reassembly)", len(got), len(want)) + } +} + +// An index with no name in the table is unsupported, never silently stored. +func TestReadFile_UnnamedMethodIndex_IsUnsupported(t *testing.T) { + p := writeMethodPak(t, []string{"Zlib"}, []zlibFixture{ + {path: "x/Y.json", blocks: [][]byte{[]byte("{}")}, method: 3}, + }) + + r, err := Open(p) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() //nolint:errcheck + + if _, err := r.ReadFile("x/Y.json"); !errors.Is(err, ErrUnsupportedFormat) { + t.Fatalf("err = %v, want ErrUnsupportedFormat", err) + } +} + +// A corrupted compressed payload must fail the entry's SHA1 gate. +func TestReadFile_ZlibCorruptPayload_FailsHashGate(t *testing.T) { + p := writeMethodPak(t, []string{"Zlib"}, []zlibFixture{ + {path: "c/D.json", blocks: [][]byte{bytes.Repeat([]byte("x"), 200)}, method: 1}, + }) + raw, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + // Flip a byte inside the first entry's compressed payload (just past its + // single-block header). + raw[compressedHeaderSize(1)+2] ^= 0xFF + if err := os.WriteFile(p, raw, 0o644); err != nil { + t.Fatal(err) + } + + r, err := Open(p) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() //nolint:errcheck + + if _, err := r.ReadFile("c/D.json"); err == nil { + t.Fatal("expected an error for a corrupted compressed payload, got nil") + } +} From 8c5969d5d8188f973026d96db1203c2b00e79265 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 10:28:50 -0400 Subject: [PATCH 26/96] feat: compile Icarus mods from the installed pak, drop the dump subsystem (#175) --- internal/core/service.go | 2 +- internal/core/service_icarus_compile_test.go | 2 +- internal/source/icarus/compile.go | 38 +- internal/source/icarus/compile_test.go | 88 +++-- internal/source/icarus/datadump.go | 311 --------------- internal/source/icarus/datadump_test.go | 382 ------------------- internal/source/icarus/helpers_test.go | 29 ++ internal/source/icarus/icarus.go | 32 +- internal/source/icarus/icarus_test.go | 28 -- internal/source/source.go | 8 +- 10 files changed, 97 insertions(+), 823 deletions(-) delete mode 100644 internal/source/icarus/datadump.go delete mode 100644 internal/source/icarus/datadump_test.go create mode 100644 internal/source/icarus/helpers_test.go diff --git a/internal/core/service.go b/internal/core/service.go index ad15736..67f6d67 100644 --- a/internal/core/service.go +++ b/internal/core/service.go @@ -517,7 +517,7 @@ func (s *Service) DownloadModToCache(ctx context.Context, gameCache *cache.Cache } destName := compiledFileName(file.FileName) destPath := filepath.Join(stagePath, destName) - if err := compiler.Compile(ctx, basePakPath, game.BaseDataPath, archivePath, destPath); err != nil { + if err := compiler.Compile(ctx, basePakPath, archivePath, destPath); err != nil { return nil, fmt.Errorf("compiling mod: %w", err) } if err := commitStagedCacheWithMarker(cachePath, stagePath, file.ID, []string{destName}); err != nil { diff --git a/internal/core/service_icarus_compile_test.go b/internal/core/service_icarus_compile_test.go index 3a7ff2a..9467438 100644 --- a/internal/core/service_icarus_compile_test.go +++ b/internal/core/service_icarus_compile_test.go @@ -53,7 +53,7 @@ func (s *fakeCompilerSource) CheckUpdates(ctx context.Context, installed []domai // through unchanged — this test only asserts Service invoked it with the // right arguments and used its output, not that it performs real PAK // compilation (Task 12 covers that). -func (s *fakeCompilerSource) Compile(ctx context.Context, basePakPath, baseDataPath, sourceFilePath, outputPath string) error { +func (s *fakeCompilerSource) Compile(ctx context.Context, basePakPath, sourceFilePath, outputPath string) error { s.compileCalls++ data, err := os.ReadFile(sourceFilePath) if err != nil { diff --git a/internal/source/icarus/compile.go b/internal/source/icarus/compile.go index 4781cc8..284965a 100644 --- a/internal/source/icarus/compile.go +++ b/internal/source/icarus/compile.go @@ -1,7 +1,6 @@ package icarus import ( - "context" "fmt" "os" "path" @@ -14,20 +13,17 @@ import ( // tables, bundles in any pre-built assets the .EXMODZ carries, and writes the // result as a new pak at outputPakPath ready to deploy as-is. // -// The base tables come from the community per-week dump (Task 12a), not from -// basePakPath: 258 of the 298 tables in a real data.pak are Oodle-compressed -// and cannot be read with the stdlib. basePakPath is still opened, for two -// things it alone can answer — which tables the installed game actually has -// (so a bare, hyphen-flattened CurrentFile resolves to a real mount path), -// and whether the dump -// is for the installed week (DumpForBuild byte-checks it against the tables -// the pak stores uncompressed). A dump that does not match fails the whole -// compile; see Task 12a. +// Base tables are read directly out of basePakPath — the installed game's own +// Content/Data/data.pak — so they are always week-correct by construction and +// the whole operation is offline. That pak stores 40 tables uncompressed and +// compresses the other 258 with Zlib, all of which internal/unrealpak reads +// with the standard library (#175). basePakPath is also what resolves a bare, +// hyphen-flattened CurrentFile to a real mount path. // -// localDumpDir is the game's optional data_dump_path: when set, base tables -// are read from that directory instead of being fetched. It is validated -// identically, so a stale local directory fails just as loudly. -func Compile(ctx context.Context, dumps *DumpStore, basePakPath, localDumpDir, exmodzPath, outputPakPath string) (err error) { +// There is no ctx parameter: every step is local file I/O over a ~2 MB pak, +// with no network call and no long-running loop to cancel. The +// source.Compiler interface still takes one, for implementations that need it. +func Compile(basePakPath, exmodzPath, outputPakPath string) (err error) { exmodzData, err := os.ReadFile(exmodzPath) if err != nil { return fmt.Errorf("icarus: reading %s: %w", exmodzPath, err) @@ -43,13 +39,6 @@ func Compile(ctx context.Context, dumps *DumpStore, basePakPath, localDumpDir, e } defer base.Close() //nolint:errcheck - // Loaded and validated before anything is written, so a week mismatch or - // an offline machine fails before a half-built pak exists on disk. - dump, err := dumps.DumpForBuild(ctx, basePakPath, localDumpDir) - if err != nil { - return err - } - out, err := unrealpak.Create(outputPakPath) if err != nil { return fmt.Errorf("icarus: creating %s: %w", outputPakPath, err) @@ -87,10 +76,9 @@ func Compile(ctx context.Context, dumps *DumpStore, basePakPath, localDumpDir, e if err != nil { return err } - baseData, ok := dump.Table(mountPath) - if !ok { - return fmt.Errorf("icarus: base data table %s is present in the installed game "+ - "but missing from the base-table dump", mountPath) + baseData, err := base.ReadFile(mountPath) + if err != nil { + return fmt.Errorf("icarus: reading base data table %s: %w", mountPath, err) } patched, err := ApplyRowPatch(baseData, row) if err != nil { diff --git a/internal/source/icarus/compile_test.go b/internal/source/icarus/compile_test.go index 9fe6503..4a0c239 100644 --- a/internal/source/icarus/compile_test.go +++ b/internal/source/icarus/compile_test.go @@ -3,9 +3,6 @@ package icarus import ( "archive/zip" "bytes" - "context" - "net/http" - "net/http/httptest" "os" "path/filepath" "strings" @@ -14,24 +11,6 @@ import ( "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" ) -// testDumpStore serves a dump containing exactly files, so validateDump agrees -// it matches the base pak built from the same map. Reuses tarGz from -// datadump_test.go (same package). -func testDumpStore(t *testing.T, files map[string][]byte) *DumpStore { - t.Helper() - entries := make(map[string]string, len(files)) - for name, data := range files { - entries[name] = string(data) - } - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write(tarGz(t, "IcarusData-test", entries)) - })) - t.Cleanup(srv.Close) - store := newDumpStore(srv.Client()) - store.treeURL = srv.URL - return store -} - func writeTestExmodzFile(t *testing.T, manifestJSON string, assets map[string][]byte) string { t.Helper() path := filepath.Join(t.TempDir(), "mod.exmodz") @@ -61,14 +40,13 @@ func TestCompile_AppliesDiffAndBundlesAssets(t *testing.T) { "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":200}}`), } basePak := writeTestBasePak(t, baseTables) - dumps := testDumpStore(t, baseTables) manifest := `{"name":"Bear Mount","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":235}]}]}` exmodzPath := writeTestExmodzFile(t, manifest, map[string][]byte{ "Bear_Mount/ASS/ITM/SK_ITM_Saddle_Bear.uasset": []byte("fake-asset"), }) outputPath := filepath.Join(t.TempDir(), "Bear_Mount_P.pak") - if err := Compile(context.Background(), dumps, basePak, "", exmodzPath, outputPath); err != nil { + if err := Compile(basePak, exmodzPath, outputPath); err != nil { t.Fatalf("Compile: %v", err) } @@ -103,14 +81,13 @@ func TestCompile_SkipsEndOfModSentinelRow(t *testing.T) { "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":200}}`), } basePak := writeTestBasePak(t, baseTables) - dumps := testDumpStore(t, baseTables) manifest := `{"name":"X","Rows":[` + `{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":235}]},` + `{"CurrentFile":"EndOfMod"}]}` exmodzPath := writeTestExmodzFile(t, manifest, nil) outputPath := filepath.Join(t.TempDir(), "out.pak") - if err := Compile(context.Background(), dumps, basePak, "", exmodzPath, outputPath); err != nil { + if err := Compile(basePak, exmodzPath, outputPath); err != nil { t.Fatalf("Compile: %v", err) } @@ -135,12 +112,11 @@ func TestCompile_RowWithoutFileItems_Errors(t *testing.T) { "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":200}}`), } basePak := writeTestBasePak(t, baseTables) - dumps := testDumpStore(t, baseTables) manifest := `{"name":"X","Rows":[{"CurrentFile":"AI-D_AIGrowth.json"}]}` exmodzPath := writeTestExmodzFile(t, manifest, nil) outputPath := filepath.Join(t.TempDir(), "out.pak") - err := Compile(context.Background(), dumps, basePak, "", exmodzPath, outputPath) + err := Compile(basePak, exmodzPath, outputPath) if err == nil { t.Fatal("expected an error for a non-sentinel row with no File_Items, got nil") } @@ -149,25 +125,55 @@ func TestCompile_RowWithoutFileItems_Errors(t *testing.T) { } } -// A stale dump must stop the compile before any output pak is written — this -// is the live case today, where the newest dump lags the installed game. -func TestCompile_DumpWeekMismatch_FailsBeforeWriting(t *testing.T) { +// The base table Compile patches must come from the installed pak itself — +// that is the whole point of the #175 pivot — so a row's output has to reflect +// the pak's own bytes, not any other source. +func TestCompile_PatchesTheBasePaksOwnTable(t *testing.T) { basePak := writeTestBasePak(t, map[string][]byte{ - "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":200}}`), - }) - dumps := testDumpStore(t, map[string][]byte{ // different week's content - "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":150}}`), + "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":200,"OnlyInPak":true}}`), }) manifest := `{"name":"X","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":235}]}]}` exmodzPath := writeTestExmodzFile(t, manifest, nil) outputPath := filepath.Join(t.TempDir(), "out.pak") - err := Compile(context.Background(), dumps, basePak, "", exmodzPath, outputPath) - if err == nil { - t.Fatal("expected an error when the dump is for a different game week, got nil") + if err := Compile(basePak, exmodzPath, outputPath); err != nil { + t.Fatalf("Compile: %v", err) + } + r, err := unrealpak.Open(outputPath) + if err != nil { + t.Fatalf("opening compiled output: %v", err) + } + defer r.Close() //nolint:errcheck + got, err := r.ReadFile("AI/D_AIGrowth.json") + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + // The patched field changed... + if !bytes.Contains(got, []byte(`"BaseMovementSpeed":235`)) { + t.Errorf("patched table = %s, want BaseMovementSpeed 235", got) + } + // ...and a field only the base pak carried survived, proving the base + // content was read from the pak rather than synthesized. + if !bytes.Contains(got, []byte(`"OnlyInPak":true`)) { + t.Errorf("patched table = %s, want the base pak's OnlyInPak field preserved", got) + } +} + +// A CurrentFile with no matching entry in the base pak fails before any output +// pak is written. +func TestCompile_UnknownBaseTable_LeavesNoOutputFile(t *testing.T) { + basePak := writeTestBasePak(t, map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{}}`), + }) + manifest := `{"name":"X","Rows":[{"CurrentFile":"AI-D_NotInPak.json","File_Items":[{"Name":"X","V":1}]}]}` + exmodzPath := writeTestExmodzFile(t, manifest, nil) + outputPath := filepath.Join(t.TempDir(), "out.pak") + + if err := Compile(basePak, exmodzPath, outputPath); err == nil { + t.Fatal("expected an error for a CurrentFile absent from the base pak, got nil") } if _, statErr := os.Stat(outputPath); statErr == nil { - t.Error("no output pak should exist after a week-mismatch failure") + t.Error("no output pak should exist after a failed compile") } } @@ -180,14 +186,13 @@ func TestCompile_UnsafeAssetPath_Errors(t *testing.T) { "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":200}}`), } basePak := writeTestBasePak(t, baseTables) - dumps := testDumpStore(t, baseTables) manifest := `{"name":"X","Rows":[]}` exmodzPath := writeTestExmodzFile(t, manifest, map[string][]byte{ "../evil.uasset": []byte("payload"), }) outputPath := filepath.Join(t.TempDir(), "out.pak") - err := Compile(context.Background(), dumps, basePak, "", exmodzPath, outputPath) + err := Compile(basePak, exmodzPath, outputPath) if err == nil { t.Fatal("expected an error for an asset path escaping the mod's own namespace, got nil") } @@ -210,14 +215,13 @@ func TestCompile_MidCompileFailure_LeavesNoOutputFile(t *testing.T) { "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":200}}`), } basePak := writeTestBasePak(t, baseTables) - dumps := testDumpStore(t, baseTables) // CurrentFile has no matching base-pak file: resolveCurrentFile fails // inside the row loop, after out has already been created. manifest := `{"name":"X","Rows":[{"CurrentFile":"AI-D_Nonexistent.json","File_Items":[{"Name":"Mount_Bear","X":1}]}]}` exmodzPath := writeTestExmodzFile(t, manifest, nil) outputPath := filepath.Join(t.TempDir(), "out.pak") - err := Compile(context.Background(), dumps, basePak, "", exmodzPath, outputPath) + err := Compile(basePak, exmodzPath, outputPath) if err == nil { t.Fatal("expected an error for an unresolvable row, got nil") } diff --git a/internal/source/icarus/datadump.go b/internal/source/icarus/datadump.go deleted file mode 100644 index d1db6ae..0000000 --- a/internal/source/icarus/datadump.go +++ /dev/null @@ -1,311 +0,0 @@ -package icarus - -import ( - "archive/tar" - "bytes" - "compress/gzip" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "io/fs" - "net/http" - "os" - "path" - "path/filepath" - "sort" - "strings" - - "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" -) - -// defaultDumpTreeURL is the community per-week unpack of Icarus's data.pak: -// https://github.com/GODOFMINECRAFT4/IcarusData. The tree is committed as -// loose JSON at the repo root, one commit per game week, with the week -// recorded only in the commit message — there are no tags or releases. This -// URL is HEAD; a specific week is addressed by substituting its commit SHA. -const defaultDumpTreeURL = "https://codeload.github.com/GODOFMINECRAFT4/IcarusData/tar.gz/refs/heads/master" - -// maxDumpBytes caps the download. The real tree is ~36 MB; this leaves room to -// grow while refusing to stream an unbounded body into memory. -const maxDumpBytes = 256 << 20 - -// maxTarEntrySize caps a single table's decompressed size. The largest real -// table (Items/D_ItemsStatic.json, per the base pak) is 7.3 MB; 64 MiB leaves -// generous headroom while refusing to trust an unbounded or lying size field -// in a tar header from a third-party, network-fetched archive. -const maxTarEntrySize = 64 << 20 - -// Build identifies the installed game, read from Icarus/Config/version.json. -// Note this carries no week number — nothing in the install does. Week -// agreement is established by content comparison, not by this value. -type Build struct { - Major, Minor, Patch int - Changelist int - DataChangelist int - FeatureLevel string -} - -func (b Build) String() string { - return fmt.Sprintf("%d.%d.%d.%d", b.Major, b.Minor, b.Patch, b.Changelist) -} - -// detectBuild reads /Icarus/Config/version.json. -func detectBuild(installRoot string) (Build, error) { - p := filepath.Join(installRoot, "Icarus", "Config", "version.json") - raw, err := os.ReadFile(p) - if err != nil { - return Build{}, fmt.Errorf("icarus: reading game version from %s: %w", p, err) - } - var doc struct { - Version struct { - Major, Minor, Patch int - Changelist int - FeatureLevel string - } - Data struct{ Changelist int } - } - if err := json.Unmarshal(raw, &doc); err != nil { - return Build{}, fmt.Errorf("icarus: parsing %s: %w", p, err) - } - return Build{ - Major: doc.Version.Major, Minor: doc.Version.Minor, Patch: doc.Version.Patch, - Changelist: doc.Version.Changelist, - DataChangelist: doc.Data.Changelist, - FeatureLevel: doc.Version.FeatureLevel, - }, nil -} - -// Dump is a fetched set of base data tables, keyed by mount-relative path -// (e.g. "Factions/D_Factions.json") with values already converted back to the -// game's CRLF line endings. -type Dump struct { - tables map[string][]byte -} - -// Table returns one table's shipped bytes. -func (d *Dump) Table(rel string) ([]byte, bool) { - b, ok := d.tables[rel] - return b, ok -} - -// DumpStore fetches base-table dumps (hosted, or from a local directory -// override — see DumpForBuild). It does not cache to disk: every call that -// doesn't supply a localDumpDir re-fetches the tree over the network. Adding -// caching is a real, tracked follow-up, not implemented here (YAGNI). -type DumpStore struct { - httpClient *http.Client - treeURL string // overridable in tests -} - -func newDumpStore(httpClient *http.Client) *DumpStore { - return &DumpStore{httpClient: httpClient, treeURL: defaultDumpTreeURL} -} - -// DumpForBuild loads the base data tables and returns them only if they match -// the installed game, proven by byte-comparing every table basePakPath stores -// uncompressed. A mismatch means the tables are for a different game week: -// that is a hard error naming the offending tables, never a silent -// best-effort. -// -// localDumpDir, when non-empty, is a user-supplied directory holding an -// unpacked data.pak JSON tree (QuickBMS output and the like); it replaces the -// network fetch entirely. Validation is the same either way — a local -// directory from the wrong week is rejected exactly like a stale hosted dump. -func (s *DumpStore) DumpForBuild(ctx context.Context, basePakPath, localDumpDir string) (*Dump, error) { - var ( - dump *Dump - err error - ) - if localDumpDir != "" { - dump, err = loadLocalDump(localDumpDir) - } else { - dump, err = s.fetchTree(ctx, s.treeURL) - } - if err != nil { - return nil, err - } - if err := validateDump(dump, basePakPath); err != nil { - if localDumpDir != "" { - return nil, fmt.Errorf("%w (tables were read from the configured data_dump_path %s)", err, localDumpDir) - } - return nil, err - } - return dump, nil -} - -// loadLocalDump reads an unpacked data.pak JSON tree from disk. The layout is -// the same one the hosted dump ships — table paths relative to the directory -// root, e.g. "Factions/D_Factions.json" — so a user can point this at QuickBMS -// output without rearranging anything. -func loadLocalDump(dir string) (*Dump, error) { - info, err := os.Stat(dir) - if err != nil { - return nil, fmt.Errorf("icarus: reading the configured data_dump_path %s: %w", dir, err) - } - if !info.IsDir() { - return nil, fmt.Errorf("icarus: the configured data_dump_path %s is not a directory", dir) - } - - dump := &Dump{tables: make(map[string][]byte)} - err = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() || !strings.HasSuffix(d.Name(), ".json") { - return nil - } - rel, err := filepath.Rel(dir, p) - if err != nil { - return err - } - body, err := os.ReadFile(p) - if err != nil { - return err - } - dump.tables[filepath.ToSlash(rel)] = toCRLF(body) - return nil - }) - if err != nil { - return nil, fmt.Errorf("icarus: scanning the configured data_dump_path %s: %w", dir, err) - } - if len(dump.tables) == 0 { - return nil, fmt.Errorf("icarus: the configured data_dump_path %s contains no JSON tables "+ - "(expected an unpacked data.pak tree, e.g. Factions/D_Factions.json)", dir) - } - return dump, nil -} - -// fetchTree downloads a dump tarball and ingests its JSON tables, restoring -// the CRLF line endings the game ships (the repo stores LF). -func (s *DumpStore) fetchTree(ctx context.Context, url string) (*Dump, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, fmt.Errorf("icarus: building dump request: %w", err) - } - resp, err := s.httpClient.Do(req) - if err != nil { - return nil, fmt.Errorf("icarus: fetching base-table dump: %w "+ - "(compiling Icarus mods requires network access — see the plan's Global Constraints)", err) - } - defer resp.Body.Close() //nolint:errcheck - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("icarus: fetching base-table dump from %s: HTTP %d", url, resp.StatusCode) - } - - zr, err := gzip.NewReader(io.LimitReader(resp.Body, maxDumpBytes)) - if err != nil { - return nil, fmt.Errorf("icarus: base-table dump is not valid gzip: %w", err) - } - defer zr.Close() //nolint:errcheck - - dump := &Dump{tables: make(map[string][]byte)} - tr := tar.NewReader(zr) - for { - hdr, err := tr.Next() - if err == io.EOF { - break - } - if err != nil { - return nil, fmt.Errorf("icarus: reading base-table dump: %w", err) - } - if hdr.Typeflag != tar.TypeReg || !strings.HasSuffix(hdr.Name, ".json") { - continue - } - // Strip the archive's single top-level directory (e.g. - // "IcarusData-/") to get the mount-relative table path. - rel := hdr.Name - if i := strings.Index(rel, "/"); i >= 0 { - rel = rel[i+1:] - } - // The repo also carries a stale "data/" copy of the tree; the - // authoritative tables are the root-level ones. - if rel == "" || strings.HasPrefix(rel, "data/") { - continue - } - if hdr.Size > maxTarEntrySize { - return nil, fmt.Errorf("icarus: base-table dump entry %s declares a %d-byte size, "+ - "exceeding the %d-byte per-table cap", rel, hdr.Size, maxTarEntrySize) - } - body, err := io.ReadAll(io.LimitReader(tr, hdr.Size)) - if err != nil { - return nil, fmt.Errorf("icarus: reading %s from base-table dump: %w", rel, err) - } - dump.tables[path.Clean(rel)] = toCRLF(body) - } - if len(dump.tables) == 0 { - return nil, fmt.Errorf("icarus: base-table dump from %s contained no JSON tables", url) - } - return dump, nil -} - -// toCRLF restores the game's line endings. The dump repo stores LF (committed -// with autocrlf); the shipped pak stores CRLF, and the two are otherwise -// byte-identical. Existing CRLFs are left alone so the conversion is -// idempotent. -func toCRLF(b []byte) []byte { - return []byte(strings.ReplaceAll(strings.ReplaceAll(string(b), "\r\n", "\n"), "\n", "\r\n")) -} - -// validateDump proves a dump belongs to the installed game. -// -// Only the tables data.pak stores *uncompressed* can be checked — the rest are -// Oodle-compressed and unreadable here, which is the whole reason the dump -// exists. That is enough: a dump built from a different week's data.pak -// disagrees on some of them, and in practice it disagrees loudly (the spike saw -// 3 differing stored tables and 6 missing tables across a 7-week gap). -func validateDump(dump *Dump, basePakPath string) error { - pak, err := unrealpak.Open(basePakPath) - if err != nil { - return fmt.Errorf("icarus: opening base pak %s for dump validation: %w", basePakPath, err) - } - defer pak.Close() //nolint:errcheck - - var missing, differing []string - checked := 0 - for _, f := range pak.Files() { - shipped, err := pak.ReadFile(f.Path) - if err != nil { - if errors.Is(err, unrealpak.ErrUnsupportedFormat) { - continue // Oodle-compressed (or similar): not readable here, and not our gate - } - // Any other ReadFile failure — corruption, a truncated payload, an - // I/O error — is not an expected skip. Silently excluding it here - // would quietly narrow what this gate actually verified, exactly - // the "no silent fallbacks" failure this function exists to prevent. - return fmt.Errorf("icarus: validating base pak %s: reading %s: %w", basePakPath, f.Path, err) - } - checked++ - got, ok := dump.Table(f.Path) - if !ok { - missing = append(missing, f.Path) - continue - } - if !bytes.Equal(got, shipped) { - differing = append(differing, f.Path) - } - } - if checked == 0 { - return fmt.Errorf("icarus: %s exposed no uncompressed tables to validate the dump against", basePakPath) - } - if len(missing) == 0 && len(differing) == 0 { - return nil - } - sort.Strings(missing) - sort.Strings(differing) - return fmt.Errorf( - "icarus: the available base-table dump does not match the installed game "+ - "(%d/%d uncompressed tables disagree: %s). The dump is for a different game week. "+ - "Wait for the dump to be updated for your game version, or roll the game back to a "+ - "matching week; compiling against a mismatched week would silently corrupt mod data", - len(missing)+len(differing), checked, summarize(append(differing, missing...))) -} - -func summarize(paths []string) string { - const max = 3 - if len(paths) <= max { - return strings.Join(paths, ", ") - } - return fmt.Sprintf("%s and %d more", strings.Join(paths[:max], ", "), len(paths)-max) -} diff --git a/internal/source/icarus/datadump_test.go b/internal/source/icarus/datadump_test.go deleted file mode 100644 index dd5f148..0000000 --- a/internal/source/icarus/datadump_test.go +++ /dev/null @@ -1,382 +0,0 @@ -package icarus - -import ( - "archive/tar" - "bytes" - "compress/gzip" - "context" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" -) - -// testStoredHeaderSize mirrors unrealpak's unexported storedHeaderSize (the -// 53-byte FPakEntry header preceding a stored file's payload). Duplicated -// here because these fixture-corruption helpers need to reach into pak bytes -// unrealpak itself does not expose, the same way reader_test.go pokes at raw -// offsets from inside package unrealpak. -const testStoredHeaderSize = 53 - -// corruptFirstEntryPayload flips the first byte of the alphabetically-first -// entry's payload (which the Writer always places at file offset -// testStoredHeaderSize, since entries are packed with no gap starting at 0). -// This breaks the entry's stored SHA1 without touching its header, producing -// a genuinely unexpected ReadFile error distinct from ErrUnsupportedFormat. -func corruptFirstEntryPayload(t *testing.T, pakPath string) { - t.Helper() - data, err := os.ReadFile(pakPath) - if err != nil { - t.Fatal(err) - } - data[testStoredHeaderSize] ^= 0xFF - if err := os.WriteFile(pakPath, data, 0o644); err != nil { - t.Fatal(err) - } -} - -// corruptFirstEntryCompressionMethod patches the alphabetically-first entry's -// on-disk header CompressionMethodIndex field (bytes 24:28 of the 53-byte -// header at file offset 0) to a nonzero value. This simulates the -// compression-refusal ReadFile takes for real Oodle-compressed entries -// without needing the Writer to emit actual compressed data, which it never -// does (it only ever produces stored, method-0 entries). -func corruptFirstEntryCompressionMethod(t *testing.T, pakPath string) { - t.Helper() - data, err := os.ReadFile(pakPath) - if err != nil { - t.Fatal(err) - } - data[24] = 1 - if err := os.WriteFile(pakPath, data, 0o644); err != nil { - t.Fatal(err) - } -} - -// writeTestBasePak builds a stored, unencrypted version-11 pak holding one -// entry per (mount-relative path, content) pair, via the Task 4 Writer. It -// stands in for Task 12's identically-named helper, which does not exist yet -// at this point in the plan's task order. -func writeTestBasePak(t *testing.T, files map[string][]byte) string { - t.Helper() - pakPath := filepath.Join(t.TempDir(), "data.pak") - w, err := unrealpak.Create(pakPath) - if err != nil { - t.Fatalf("creating test base pak: %v", err) - } - for rel, data := range files { - if err := w.AddFile(rel, data); err != nil { - t.Fatalf("AddFile(%q): %v", rel, err) - } - } - if err := w.Close(); err != nil { - t.Fatalf("closing test base pak: %v", err) - } - return pakPath -} - -// tarGz builds a dump-shaped tarball: a single top-level directory, then the -// table tree beneath it, LF-terminated exactly as the real repo stores it. -func tarGz(t *testing.T, root string, files map[string]string) []byte { - t.Helper() - var buf bytes.Buffer - zw := gzip.NewWriter(&buf) - tw := tar.NewWriter(zw) - for name, body := range files { - hdr := &tar.Header{Name: root + "/" + name, Mode: 0o644, Size: int64(len(body))} - if err := tw.WriteHeader(hdr); err != nil { - t.Fatal(err) - } - if _, err := tw.Write([]byte(body)); err != nil { - t.Fatal(err) - } - } - if err := tw.Close(); err != nil { - t.Fatal(err) - } - if err := zw.Close(); err != nil { - t.Fatal(err) - } - return buf.Bytes() -} - -func TestDetectBuild_ReadsVersionJSON(t *testing.T) { - root := t.TempDir() - cfg := filepath.Join(root, "Icarus", "Config") - if err := os.MkdirAll(cfg, 0o755); err != nil { - t.Fatal(err) - } - const vjson = `{"Name":"Icarus","Version":{"Major":3,"Minor":0,"Patch":21,` + - `"Changelist":155335,"BuildType":"Shipping","FeatureLevel":"DangerousHorizons"},` + - `"Data":{"Changelist":155151}}` - if err := os.WriteFile(filepath.Join(cfg, "version.json"), []byte(vjson), 0o644); err != nil { - t.Fatal(err) - } - - b, err := detectBuild(root) - if err != nil { - t.Fatalf("detectBuild: %v", err) - } - if got := b.String(); got != "3.0.21.155335" { - t.Errorf("Build.String() = %q, want 3.0.21.155335", got) - } - if b.DataChangelist != 155151 { - t.Errorf("DataChangelist = %d, want 155151", b.DataChangelist) - } -} - -func TestDetectBuild_MissingVersionFile_Errors(t *testing.T) { - if _, err := detectBuild(t.TempDir()); err == nil { - t.Fatal("expected error when version.json is absent, got nil") - } -} - -// A dump whose stored tables match the local pak byte-for-byte (after CRLF -// restoration) is accepted, and its tables are exposed with shipped bytes. -func TestDumpStore_DumpForBuild_AcceptsMatchingDump(t *testing.T) { - const rel = "Factions/D_Factions.json" - shipped := []byte("{\r\n \"Rows\": []\r\n}") // CRLF, as the pak stores it - dumped := "{\n \"Rows\": []\n}" // LF, as the repo stores it - - pak := writeTestBasePak(t, map[string][]byte{rel: shipped}) // Task 12's helper - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write(tarGz(t, "IcarusData-abc123", map[string]string{rel: dumped})) - })) - defer srv.Close() - - store := newDumpStore(srv.Client()) - store.treeURL = srv.URL // test seam - - dump, err := store.DumpForBuild(context.Background(), pak, "") - if err != nil { - t.Fatalf("DumpForBuild: %v", err) - } - got, ok := dump.Table(rel) - if !ok { - t.Fatalf("dump has no table %q", rel) - } - if !bytes.Equal(got, shipped) { - t.Errorf("table bytes = %q, want the shipped CRLF form %q", got, shipped) - } -} - -// The case that is live today: the newest dump is an older week than the -// install. Must fail loudly and name what disagreed. -func TestDumpStore_DumpForBuild_RejectsWrongWeek(t *testing.T) { - const rel = "Factions/D_Factions.json" - pak := writeTestBasePak(t, map[string][]byte{rel: []byte("{\r\n \"Rows\": [1]\r\n}")}) - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write(tarGz(t, "IcarusData-old", map[string]string{rel: "{\n \"Rows\": []\n}"})) - })) - defer srv.Close() - - store := newDumpStore(srv.Client()) - store.treeURL = srv.URL - - _, err := store.DumpForBuild(context.Background(), pak, "") - if err == nil { - t.Fatal("expected an error for a dump that does not match the install, got nil") - } - if !strings.Contains(err.Error(), rel) { - t.Errorf("error %q should name the table that disagreed (%s)", err, rel) - } -} - -// writeLocalDump lays out an unpacked-data.pak-shaped directory on disk. -func writeLocalDump(t *testing.T, files map[string]string) string { - t.Helper() - dir := t.TempDir() - for rel, body := range files { - full := filepath.Join(dir, filepath.FromSlash(rel)) - if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(full, []byte(body), 0o644); err != nil { - t.Fatal(err) - } - } - return dir -} - -// With a local dump directory configured, the network is never touched. -func TestDumpStore_DumpForBuild_LocalDirOverridesFetch(t *testing.T) { - const rel = "Factions/D_Factions.json" - shipped := []byte("{\r\n \"Rows\": []\r\n}") - pak := writeTestBasePak(t, map[string][]byte{rel: shipped}) - local := writeLocalDump(t, map[string]string{rel: "{\n \"Rows\": []\n}"}) - - fetched := false - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fetched = true - w.WriteHeader(http.StatusInternalServerError) - })) - defer srv.Close() - store := newDumpStore(srv.Client()) - store.treeURL = srv.URL - - dump, err := store.DumpForBuild(context.Background(), pak, local) - if err != nil { - t.Fatalf("DumpForBuild with a local dump dir: %v", err) - } - if fetched { - t.Error("the hosted dump was fetched even though a local dump dir was configured") - } - got, ok := dump.Table(rel) - if !ok || !bytes.Equal(got, shipped) { - t.Errorf("table bytes = %q (found=%v), want the shipped CRLF form %q", got, ok, shipped) - } -} - -// A local directory already storing CRLF must load unchanged — QuickBMS writes -// whatever the pak stored, so the conversion has to be idempotent. -func TestDumpStore_DumpForBuild_LocalDirAlreadyCRLF(t *testing.T) { - const rel = "Factions/D_Factions.json" - shipped := "{\r\n \"Rows\": []\r\n}" - pak := writeTestBasePak(t, map[string][]byte{rel: []byte(shipped)}) - local := writeLocalDump(t, map[string]string{rel: shipped}) - - store := newDumpStore(http.DefaultClient) - store.treeURL = "http://127.0.0.1:0/never-used" - - if _, err := store.DumpForBuild(context.Background(), pak, local); err != nil { - t.Fatalf("DumpForBuild with a CRLF local dump dir: %v", err) - } -} - -// A local dir from the wrong week is rejected exactly like a stale hosted -// dump, and the error points at the configured path. -func TestDumpStore_DumpForBuild_LocalDirWrongWeek_Rejected(t *testing.T) { - const rel = "Factions/D_Factions.json" - pak := writeTestBasePak(t, map[string][]byte{rel: []byte("{\r\n \"Rows\": [1]\r\n}")}) - local := writeLocalDump(t, map[string]string{rel: "{\n \"Rows\": []\n}"}) - - store := newDumpStore(http.DefaultClient) - store.treeURL = "http://127.0.0.1:0/never-used" - - _, err := store.DumpForBuild(context.Background(), pak, local) - if err == nil { - t.Fatal("expected an error for a local dump dir from a different week, got nil") - } - if !strings.Contains(err.Error(), rel) { - t.Errorf("error %q should name the disagreeing table (%s)", err, rel) - } - if !strings.Contains(err.Error(), local) { - t.Errorf("error %q should name the configured data_dump_path (%s)", err, local) - } -} - -func TestDumpStore_DumpForBuild_LocalDirEmpty_IsActionable(t *testing.T) { - pak := writeTestBasePak(t, map[string][]byte{"a/B.json": []byte("{}")}) - store := newDumpStore(http.DefaultClient) - - _, err := store.DumpForBuild(context.Background(), pak, t.TempDir()) - if err == nil { - t.Fatal("expected an error for a data_dump_path holding no JSON tables, got nil") - } -} - -func TestDumpStore_DumpForBuild_NetworkFailure_IsActionable(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - })) - defer srv.Close() - - store := newDumpStore(srv.Client()) - store.treeURL = srv.URL - - pak := writeTestBasePak(t, map[string][]byte{"a/B.json": []byte("{}")}) - _, err := store.DumpForBuild(context.Background(), pak, "") - if err == nil { - t.Fatal("expected an error when the dump host fails, got nil") - } -} - -// A tar entry whose declared header size exceeds the per-table cap must be -// rejected before any content is read — guards against a network-fetched, -// third-party archive with a corrupt or lying size field driving an -// unbounded allocation. The fixture never writes real content matching the -// declared size (impractical at 64+ MiB): tw.WriteHeader alone already -// produces a header a tar.Reader can parse, and the cap fires on hdr.Size -// alone, before fetchTree ever attempts to read the entry's body. -func TestDumpStore_DumpForBuild_RejectsOversizedTarEntry(t *testing.T) { - var buf bytes.Buffer - gz := gzip.NewWriter(&buf) - tw := tar.NewWriter(gz) - hdr := &tar.Header{ - Name: "IcarusData-test/Huge/D_Huge.json", - Mode: 0o644, - Size: maxTarEntrySize + 1, - } - if err := tw.WriteHeader(hdr); err != nil { - t.Fatal(err) - } - // Deliberately not calling tw.Close() or writing the declared body. - if err := gz.Close(); err != nil { - t.Fatal(err) - } - - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write(buf.Bytes()) - })) - defer srv.Close() - - store := newDumpStore(srv.Client()) - store.treeURL = srv.URL - - pak := writeTestBasePak(t, map[string][]byte{"a/B.json": []byte("{}")}) - _, err := store.DumpForBuild(context.Background(), pak, "") - if err == nil { - t.Fatal("expected an error for an oversized tar entry, got nil") - } - if !strings.Contains(err.Error(), "Huge/D_Huge.json") { - t.Errorf("error %q should name the offending entry", err) - } -} - -// A base pak entry that fails to read for a reason OTHER than -// unrealpak.ErrUnsupportedFormat (corruption, a truncated payload, an I/O -// error) must fail validateDump loudly, not be silently folded into the -// "not our gate" skip that Oodle-compressed entries get. -func TestValidateDump_CorruptedStoredEntry_FailsLoudly(t *testing.T) { - const rel = "a/B.json" - pak := writeTestBasePak(t, map[string][]byte{rel: []byte("{\r\n}")}) - corruptFirstEntryPayload(t, pak) - - dump := &Dump{tables: map[string][]byte{rel: []byte("{\r\n}")}} - err := validateDump(dump, pak) - if err == nil { - t.Fatal("expected an error for a corrupted stored entry, got nil") - } - if strings.Contains(err.Error(), "different game week") { - t.Errorf("error %q should report the read failure, not the mismatch/wrong-week message "+ - "(a corrupted entry is not evidence of a stale dump)", err) - } - if !strings.Contains(err.Error(), rel) { - t.Errorf("error %q should name the unreadable table (%s)", err, rel) - } -} - -// An entry that refuses with unrealpak.ErrUnsupportedFormat (the real-world -// case: Oodle compression) must still be skipped, not treated as a -// validateDump failure — it is excluded from the check, not a reason to -// reject the dump. -func TestValidateDump_SkipsUnsupportedFormatEntry_NotAnError(t *testing.T) { - const compressedRel = "a/Apple.json" // sorts first -> lands at file offset 0 - const okRel = "z/Zebra.json" - okShipped := []byte("{\r\n \"Rows\": []\r\n}") - - pak := writeTestBasePak(t, map[string][]byte{ - compressedRel: []byte("{\r\n}"), - okRel: okShipped, - }) - corruptFirstEntryCompressionMethod(t, pak) - - dump := &Dump{tables: map[string][]byte{okRel: okShipped}} - if err := validateDump(dump, pak); err != nil { - t.Fatalf("validateDump with one ErrUnsupportedFormat entry and one matching entry: %v", err) - } -} diff --git a/internal/source/icarus/helpers_test.go b/internal/source/icarus/helpers_test.go new file mode 100644 index 0000000..14e5bb9 --- /dev/null +++ b/internal/source/icarus/helpers_test.go @@ -0,0 +1,29 @@ +package icarus + +import ( + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" +) + +// writeTestBasePak builds a stored, unencrypted version-11 pak holding one +// entry per (mount-relative path, content) pair, via the Task 4 Writer. It is +// the shared fixture builder for this package's compile tests. +func writeTestBasePak(t *testing.T, files map[string][]byte) string { + t.Helper() + pakPath := filepath.Join(t.TempDir(), "data.pak") + w, err := unrealpak.Create(pakPath) + if err != nil { + t.Fatalf("creating test base pak: %v", err) + } + for rel, data := range files { + if err := w.AddFile(rel, data); err != nil { + t.Fatalf("AddFile(%q): %v", rel, err) + } + } + if err := w.Close(); err != nil { + t.Fatalf("closing test base pak: %v", err) + } + return pakPath +} diff --git a/internal/source/icarus/icarus.go b/internal/source/icarus/icarus.go index 7102d91..029505d 100644 --- a/internal/source/icarus/icarus.go +++ b/internal/source/icarus/icarus.go @@ -20,7 +20,6 @@ const gameID = "icarus" // API described in docs/plans/2026-07-29-icarus-exmod-pak-research.md. type Icarus struct { firestore *firestoreClient - dumps *DumpStore // nil until SetDataDir is called } // New constructs an Icarus source. projectID is the Firestore project ID @@ -31,23 +30,6 @@ func New(httpClient *http.Client, projectID string) *Icarus { return &Icarus{firestore: newFirestoreClient(projectID, httpClient)} } -// SetDataDir constructs the base-table dump store once the service's data -// directory is known, gating Compile on it having been called at all (see -// TestIcarus_Compile_WithoutDataDir_FailsLoudly) — DumpStore itself has no -// current use for dataDir's value (it fetches on demand rather than caching -// to disk, see DumpStore's doc comment), so the parameter exists only to -// satisfy the shared `interface{ SetDataDir(string) }` duck-typed contract -// cmd/lmm/root.go's registerSource calls uniformly across sources. This is a -// post-construction setter rather than a New parameter because Task 8 froze -// New(httpClient, projectID) at exactly those two params — Task 9's call -// site already depends on that signature — so the data dir arrives the same -// way API keys do: an optional setter the registration pipeline calls when -// present (mirroring its existing SetAPIKey wiring). -func (s *Icarus) SetDataDir(dataDir string) { - _ = dataDir // unused: see doc comment above - s.dumps = newDumpStore(s.firestore.httpClient) -} - var ( _ source.ModSource = (*Icarus)(nil) _ source.CapabilityReporter = (*Icarus)(nil) @@ -55,16 +37,10 @@ var ( ) // Compile implements source.Compiler by delegating to the package-level -// Compile function (Task 12) — basePakPath/baseDataPath/sourceFilePath/ -// outputPath map directly onto Compile's basePakPath/localDumpDir/exmodzPath/ -// outputPakPath parameters. The base-table dump store (Task 12a) is supplied -// from the source itself; the per-game dump-directory override arrives as -// baseDataPath, since only the caller has the game's config. -func (s *Icarus) Compile(ctx context.Context, basePakPath, baseDataPath, sourceFilePath, outputPath string) error { - if s.dumps == nil { - return fmt.Errorf("source %q: not initialized with a data directory (SetDataDir was never called)", s.ID()) - } - return Compile(ctx, s.dumps, basePakPath, baseDataPath, sourceFilePath, outputPath) +// Compile function. ctx is unused: compiling is pure local file I/O against +// the installed game's own pak (#175), with nothing to cancel. +func (s *Icarus) Compile(_ context.Context, basePakPath, sourceFilePath, outputPath string) error { + return Compile(basePakPath, sourceFilePath, outputPath) } func (s *Icarus) ID() string { return "icarus" } diff --git a/internal/source/icarus/icarus_test.go b/internal/source/icarus/icarus_test.go index abf45bd..399f45e 100644 --- a/internal/source/icarus/icarus_test.go +++ b/internal/source/icarus/icarus_test.go @@ -5,7 +5,6 @@ import ( "encoding/json" "net/http" "net/http/httptest" - "strings" "testing" "github.com/DonovanMods/linux-mod-manager/internal/domain" @@ -135,30 +134,3 @@ func TestFileNameFromURL(t *testing.T) { }) } } - -// TestIcarus_Compile_WithoutDataDir_FailsLoudly pins that a source -// constructed via New but never wired with SetDataDir (e.g. a registration -// path that forgets the optional-setter call) fails loudly instead of -// panicking on a nil dumps store. -func TestIcarus_Compile_WithoutDataDir_FailsLoudly(t *testing.T) { - src := New(nil, "test-project") - - err := src.Compile(context.Background(), "/base.pak", "", "/mod.exmodz", "/out.pak") - if err == nil { - t.Fatal("Compile: expected an error when SetDataDir was never called") - } - if !strings.Contains(err.Error(), "SetDataDir") { - t.Errorf("Compile error = %q, want it to mention SetDataDir", err.Error()) - } -} - -// TestIcarus_SetDataDir_ConstructsDumpStore pins that SetDataDir wires a -// non-nil dumps store, so a real registration call unblocks Compile. -func TestIcarus_SetDataDir_ConstructsDumpStore(t *testing.T) { - src := New(nil, "test-project") - src.SetDataDir(t.TempDir()) - - if src.dumps == nil { - t.Fatal("SetDataDir: dumps store still nil") - } -} diff --git a/internal/source/source.go b/internal/source/source.go index 622d5f6..ad95693 100644 --- a/internal/source/source.go +++ b/internal/source/source.go @@ -150,11 +150,9 @@ type DownloadHeaderProvider interface { // replaces the downloaded file in cache, so everything downstream (Install, // the linker) treats it exactly like a DeployCopy file. // -// basePakPath and baseDataPath are both resolved by the caller from the game's -// config: basePakPath from game.InstallPath, baseDataPath from the game's -// optional data_dump_path ("" when unset — see Step 6b). sourceFilePath is the -// just-downloaded file; outputPath is where the compiled result must be +// basePakPath is resolved by the caller from game.InstallPath; sourceFilePath +// is the just-downloaded file; outputPath is where the compiled result must be // written. type Compiler interface { - Compile(ctx context.Context, basePakPath, baseDataPath, sourceFilePath, outputPath string) error + Compile(ctx context.Context, basePakPath, sourceFilePath, outputPath string) error } From 8ead12011f43e869ffd427e30757385a3dedb99a Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 10:36:47 -0400 Subject: [PATCH 27/96] refactor: drop the data_dump_path setting and SetDataDir wiring (#175) --- cmd/lmm/root.go | 29 ++++++------------ cmd/lmm/root_test.go | 42 ++++----------------------- internal/domain/game.go | 3 -- internal/storage/config/games.go | 29 +++++++++--------- internal/storage/config/games_test.go | 24 --------------- 5 files changed, 28 insertions(+), 99 deletions(-) delete mode 100644 internal/storage/config/games_test.go diff --git a/cmd/lmm/root.go b/cmd/lmm/root.go index b42d48b..d3f9b33 100644 --- a/cmd/lmm/root.go +++ b/cmd/lmm/root.go @@ -203,7 +203,7 @@ func initService() (*core.Service, error) { } // Register mod sources - registerSources(svc, cfg.ConfigDir, cfg.DataDir) + registerSources(svc, cfg.ConfigDir) return svc, nil } @@ -226,29 +226,21 @@ var builtinSourceFactories = []func() source.ModSource{ // registerSources registers all available mod sources with the service // through one ordered pipeline: built-ins first (so the collision rule's // "first wins" preserves their identity against a same-id custom -// definition), then user-defined sources from /sources/. dataDir -// is threaded through to registerSource for DeployCompile sources (#136 -// Task 13) that need it wired via the SetDataDir optional setter. -func registerSources(svc *core.Service, cfgDir, dataDir string) { +// definition), then user-defined sources from /sources/. +func registerSources(svc *core.Service, cfgDir string) { for _, factory := range builtinSourceFactories { - registerSource(svc, factory(), dataDir) + registerSource(svc, factory()) } - registerCustomSources(svc, cfgDir, dataDir) + registerCustomSources(svc, cfgDir) } // registerSource runs src through the shared registration steps used for // both built-in and custom sources: collision check (first registration // wins, warning on customSourceWarnWriter) → API-key resolution (env var via // envKeyFor, falling back to the stored DB token) → SetAPIKey when the -// source accepts one → SetDataDir when the source accepts one (Icarus's -// Compile is gated on SetDataDir having been called at all: that call -// constructs the base-table dump store. dataDir's value itself is currently -// unused there — that store fetches on demand rather than caching to disk, -// #136 review round 3 — SetDataDir just fulfils the shared interface. New -// itself can't take dataDir since Task 8/9 froze its 2-arg signature) → -// RegisterSource. -func registerSource(svc *core.Service, src source.ModSource, dataDir string) { +// source accepts one → RegisterSource. +func registerSource(svc *core.Service, src source.ModSource) { id := src.ID() // Custom sources are constructed (custom.New) by the caller before this // runs; a definition that both collides with an existing ID AND fails to @@ -267,9 +259,6 @@ func registerSource(svc *core.Service, src source.ModSource, dataDir string) { setter.SetAPIKey(key) } } - if setter, ok := src.(interface{ SetDataDir(string) }); ok { - setter.SetDataDir(dataDir) - } svc.RegisterSource(src) } @@ -306,7 +295,7 @@ func customSourceWarnWriter() io.Writer { // registerCustomSources loads user-defined source definitions and registers // the valid ones. Broken definitions warn (via customSourceWarnWriter, normally // os.Stderr) and are skipped — a bad file must never prevent lmm from starting. -func registerCustomSources(svc *core.Service, cfgDir, dataDir string) { +func registerCustomSources(svc *core.Service, cfgDir string) { defs, loadErrs, err := config.LoadSourceDefinitions(cfgDir) if err != nil { fmt.Fprintf(customSourceWarnWriter(), "warning: loading custom sources: %v\n", err) @@ -321,7 +310,7 @@ func registerCustomSources(svc *core.Service, cfgDir, dataDir string) { fmt.Fprintf(customSourceWarnWriter(), "warning: skipping source %q: %v\n", def.ID, err) continue } - registerSource(svc, src, dataDir) + registerSource(svc, src) } } diff --git a/cmd/lmm/root_test.go b/cmd/lmm/root_test.go index 62e45f7..6fd6797 100644 --- a/cmd/lmm/root_test.go +++ b/cmd/lmm/root_test.go @@ -48,7 +48,7 @@ func TestRegisterSources_BuiltinStillAuthenticatesWithEnvAndToken(t *testing.T) t.Cleanup(func() { require.NoError(t, svc.Close()) }) require.NoError(t, svc.SaveSourceToken("nexusmods", "stored-db-key")) - registerSources(svc, t.TempDir(), t.TempDir()) + registerSources(svc, t.TempDir()) src, err := svc.GetSource("nexusmods") require.NoError(t, err) @@ -101,7 +101,7 @@ func TestRegisterSource_KeyResolutionPrecedence(t *testing.T) { mockAuthSource: mockAuthSource{id: "precedence-src", name: "Precedence Src"}, envKey: envVar, } - registerSource(svc, mock, t.TempDir()) + registerSource(svc, mock) assert.Equal(t, "env-value", mock.apiKey, "env var must take precedence over a stored DB token") }) @@ -115,42 +115,12 @@ func TestRegisterSource_KeyResolutionPrecedence(t *testing.T) { mockAuthSource: mockAuthSource{id: "precedence-src", name: "Precedence Src"}, envKey: envVar, } - registerSource(svc, mock, t.TempDir()) + registerSource(svc, mock) assert.Equal(t, "token-value", mock.apiKey, "stored token must apply when no env var is set") }) } -// recordingDataDirSource is a mockAuthSource that also implements the -// optional SetDataDir(string) setter (icarus.Icarus, #136 Task 13), so -// TestRegisterSource_WiresDataDir can pin that registerSource calls it with -// the resolved data directory - the same optional-setter pattern SetAPIKey -// already uses, just for a different capability. -type recordingDataDirSource struct { - mockAuthSource - dataDir string -} - -func (r *recordingDataDirSource) SetDataDir(dataDir string) { r.dataDir = dataDir } - -// TestRegisterSource_WiresDataDir pins that registerSource calls SetDataDir -// on a source that implements it, passing through the exact dataDir it was -// given - the seam icarus.Icarus.SetDataDir relies on to ever get a working -// dumps store outside of a test that constructs Icarus directly. -func TestRegisterSource_WiresDataDir(t *testing.T) { - svc, err := core.NewService(core.ServiceConfig{ - ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir(), - }) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, svc.Close()) }) - - mock := &recordingDataDirSource{mockAuthSource: mockAuthSource{id: "data-dir-src", name: "Data Dir Src"}} - wantDataDir := t.TempDir() - registerSource(svc, mock, wantDataDir) - - assert.Equal(t, wantDataDir, mock.dataDir, "registerSource must pass its dataDir through to SetDataDir") -} - // TestRegisterSources_DerivedEnvKeyForCustom pins that a custom source with // no EnvKeyProvider still resolves its key via the derived LMM__API_KEY // convention (envKeyFor's fallback to envKeyForSourceID) through the unified @@ -180,7 +150,7 @@ manifest: `) t.Setenv("LMM_MY_CUSTOM_API_KEY", "custom-env-key") - registerSources(svc, cfgDir, t.TempDir()) + registerSources(svc, cfgDir) src, err := svc.GetSource("my-custom") require.NoError(t, err) @@ -214,7 +184,7 @@ directory: customSourceWarnOut = &warnBuf t.Cleanup(func() { customSourceWarnOut = nil }) - registerSources(svc, cfgDir, t.TempDir()) + registerSources(svc, cfgDir) src, err := svc.GetSource("nexusmods") require.NoError(t, err) @@ -360,7 +330,7 @@ directory: path: /this/path/should/not/exist/lmm-test-fixture `) // construction-failure branch: Validate passes, NewDirectory's os.Stat fails - registerCustomSources(svc, cfgDir, t.TempDir()) + registerCustomSources(svc, cfgDir) sources := svc.ListSources() byID := make(map[string]source.ModSource, len(sources)) diff --git a/internal/domain/game.go b/internal/domain/game.go index 6166266..57f2107 100644 --- a/internal/domain/game.go +++ b/internal/domain/game.go @@ -46,9 +46,6 @@ type Game struct { CachePath string // Optional: custom cache path for this game's mods Hooks GameHooks // Optional: hooks for install/uninstall operations DeployMode DeployMode // How to handle downloaded files (extract vs copy) - // BaseDataPath is optional: a directory holding an unpacked data.pak JSON - // tree, used instead of fetching the hosted base-table dump (compile games only) - BaseDataPath string } // DeployMode determines how downloaded mod archives are handled diff --git a/internal/storage/config/games.go b/internal/storage/config/games.go index 8860115..ee7d142 100644 --- a/internal/storage/config/games.go +++ b/internal/storage/config/games.go @@ -49,15 +49,14 @@ type GameHooksYAML struct { // GameConfig is the YAML representation of a game type GameConfig struct { - Name string `yaml:"name"` - InstallPath string `yaml:"install_path"` - ModPath string `yaml:"mod_path"` - Sources map[string]string `yaml:"sources"` - LinkMethod string `yaml:"link_method,omitempty"` - CachePath string `yaml:"cache_path,omitempty"` - Hooks GameHooksYAML `yaml:"hooks,omitempty"` - DeployMode string `yaml:"deploy_mode,omitempty"` - BaseDataPath string `yaml:"data_dump_path,omitempty"` + Name string `yaml:"name"` + InstallPath string `yaml:"install_path"` + ModPath string `yaml:"mod_path"` + Sources map[string]string `yaml:"sources"` + LinkMethod string `yaml:"link_method,omitempty"` + CachePath string `yaml:"cache_path,omitempty"` + Hooks GameHooksYAML `yaml:"hooks,omitempty"` + DeployMode string `yaml:"deploy_mode,omitempty"` } // GamesFile is the top-level games.yaml structure @@ -98,7 +97,6 @@ func loadGamesLocked(configDir string) (map[string]*domain.Game, error) { LinkMethodExplicit: cfg.LinkMethod != "", CachePath: ExpandPath(cfg.CachePath), DeployMode: domain.ParseDeployMode(cfg.DeployMode), - BaseDataPath: ExpandPath(cfg.BaseDataPath), Hooks: domain.GameHooks{ Install: domain.HookConfig{ BeforeAll: ExpandPath(cfg.Hooks.Install.BeforeAll), @@ -136,12 +134,11 @@ func saveGamesLocked(configDir string, games map[string]*domain.Game) error { for id, game := range games { cfg := GameConfig{ - Name: game.Name, - InstallPath: game.InstallPath, - ModPath: game.ModPath, - Sources: game.SourceIDs, - CachePath: game.CachePath, - BaseDataPath: game.BaseDataPath, + Name: game.Name, + InstallPath: game.InstallPath, + ModPath: game.ModPath, + Sources: game.SourceIDs, + CachePath: game.CachePath, Hooks: GameHooksYAML{ Install: HookConfigYAML{ BeforeAll: game.Hooks.Install.BeforeAll, diff --git a/internal/storage/config/games_test.go b/internal/storage/config/games_test.go deleted file mode 100644 index 30fe64a..0000000 --- a/internal/storage/config/games_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package config - -import ( - "os" - "path/filepath" - "testing" -) - -func TestLoadGames_DataDumpPath(t *testing.T) { - dir := t.TempDir() - yaml := "games:\n icarus:\n name: Icarus\n install_path: /games/icarus\n" + - " mod_path: /games/icarus/mods\n data_dump_path: /dumps/week243\n" - if err := os.WriteFile(filepath.Join(dir, "games.yaml"), []byte(yaml), 0o644); err != nil { - t.Fatal(err) - } - - games, err := LoadGames(dir) - if err != nil { - t.Fatalf("LoadGames: %v", err) - } - if got := games["icarus"].BaseDataPath; got != "/dumps/week243" { - t.Errorf("BaseDataPath = %q, want /dumps/week243", got) - } -} From d4008344aa5a99222c66493196a2a44c74d73698 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 10:42:53 -0400 Subject: [PATCH 28/96] docs: drop data_dump_path, document compiling from the installed pak (#175) --- CHANGELOG.md | 2 +- README.md | 2 -- docs/configuration.md | 23 +++++++++++------------ 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b6ca11..d64762d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- **Icarus built-in mod source** (`internal/source/icarus`): a public, unauthenticated Firestore-backed catalog (Project Daedalus) — `lmm search`/`install`/`update` work against it like NexusMods/CurseForge. A `.exmodz` mod file now compiles into a deployable `_P.pak` at download time via a new, game-agnostic `internal/unrealpak` PAK reader/writer and the new `deploy_mode: compile` game setting; a plain `.pak` file from the same catalog is unaffected and deploys through the existing extract/copy pipeline unchanged. An optional per-game `data_dump_path` points compilation at your own unpacked `data.pak` JSON tree instead of fetching the hosted community base-table dump — both are `games.yaml`-only settings, with no new CLI flag or TUI screen (#136) +- **Icarus built-in mod source** (`internal/source/icarus`): a public, unauthenticated Firestore-backed catalog (Project Daedalus) — `lmm search`/`install`/`update` work against it like NexusMods/CurseForge. A `.exmodz` mod file now compiles into a deployable `_P.pak` at download time via a new, game-agnostic `internal/unrealpak` PAK reader/writer and the new `deploy_mode: compile` game setting; a plain `.pak` file from the same catalog is unaffected and deploys through the existing extract/copy pipeline unchanged. Base data tables are read directly from the installed game's own `data.pak`, so a compile always matches the installed game version and works entirely offline; `internal/unrealpak` reads both the stored and the Zlib-compressed entries that pak contains, using only the standard library (#136, #175) ## [1.27.1] - 2026-07-30 diff --git a/README.md b/README.md index f743786..2717a8a 100644 --- a/README.md +++ b/README.md @@ -427,8 +427,6 @@ games: install_path: "/path/to/Steam/steamapps/common/Icarus" mod_path: "/path/to/Steam/steamapps/common/Icarus/Icarus/Content/Paks/mods" deploy_mode: compile - # data_dump_path: ~/icarus-data-dump # Optional: compile from your own - # unpacked data.pak JSON tree instead of the hosted community dump sources: icarus: "icarus" ``` diff --git a/docs/configuration.md b/docs/configuration.md index b49d6ba..b40cb57 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -20,17 +20,16 @@ Defines moddable games. Each game is keyed by a unique slug (e.g. `skyrim-se`). ### Game options -| Option | Type | Required | Description | -| ---------------- | ------ | -------- | ------------------------------------------------------------------------------------------------ | -| `name` | string | yes | Display name | -| `install_path` | string | yes | Game installation directory (supports `~`) | -| `mod_path` | string | yes | Directory where mods are deployed (supports `~`) | -| `sources` | map | yes | Source ID to game ID mapping (see below) | -| `link_method` | string | no | Override global link method: `symlink`, `hardlink`, `copy` | -| `cache_path` | string | no | Per-game cache directory override | -| `hooks` | object | no | Scripts to run around install/uninstall (see below) | -| `deploy_mode` | string | no | How to handle mod archives: `extract` (default), `copy`, or `compile` | -| `data_dump_path` | string | no | Compile-mode only: local unpacked data.pak JSON tree, used instead of the hosted base-table dump | +| Option | Type | Required | Description | +| -------------- | ------ | -------- | --------------------------------------------------------------------- | +| `name` | string | yes | Display name | +| `install_path` | string | yes | Game installation directory (supports `~`) | +| `mod_path` | string | yes | Directory where mods are deployed (supports `~`) | +| `sources` | map | yes | Source ID to game ID mapping (see below) | +| `link_method` | string | no | Override global link method: `symlink`, `hardlink`, `copy` | +| `cache_path` | string | no | Per-game cache directory override | +| `hooks` | object | no | Scripts to run around install/uninstall (see below) | +| `deploy_mode` | string | no | How to handle mod archives: `extract` (default), `copy`, or `compile` | ### Hooks (games.yaml) @@ -58,7 +57,7 @@ The `deploy_mode` option controls how downloaded mod archives are handled: - **`extract`** (default): Archives are extracted to the mod path. Use for games where mods are loose files (e.g., Skyrim, Fallout). - **`copy`**: Archives are copied as-is to the mod path without extraction. Use for games that expect mod files to remain as archives (e.g., Minecraft `.jar` files, some Unity games). -- **`compile`**: The downloaded file is compiled into a new artifact before caching (currently Icarus only: an `.exmodz` diff is applied to the game's base data tables to produce a deployable `_P.pak`). Only sources that implement compiling support this mode. Optional `data_dump_path` points compilation at your own unpacked `data.pak` JSON tree instead of the hosted community dump; it must match the installed game version, and a mismatch is a hard error. +- **`compile`**: The downloaded file is compiled into a new artifact before caching (currently Icarus only: an `.exmodz` diff is applied to the game's base data tables to produce a deployable `_P.pak`). Only sources that implement compiling support this mode. The base data tables are read directly from the installed game's own `data.pak`, so a compile always matches the installed game version and needs no network access. Example: From ef687fdc1a39dce09fe112cba089e22f07a299ab Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 11:01:41 -0400 Subject: [PATCH 29/96] fix: apply .EXMOD rows to the real DataTable schema as upserts (#175) --- internal/source/icarus/compile_test.go | 6 +- internal/source/icarus/exmod.go | 84 ++++++++-- internal/source/icarus/exmod_test.go | 207 ++++++++++++++++++++++--- 3 files changed, 258 insertions(+), 39 deletions(-) diff --git a/internal/source/icarus/compile_test.go b/internal/source/icarus/compile_test.go index 4a0c239..433b274 100644 --- a/internal/source/icarus/compile_test.go +++ b/internal/source/icarus/compile_test.go @@ -37,7 +37,7 @@ func writeTestExmodzFile(t *testing.T, manifestJSON string, assets map[string][] // the original brief's fixtures assumed. See task-12-report.md "plan delta". func TestCompile_AppliesDiffAndBundlesAssets(t *testing.T) { baseTables := map[string][]byte{ - "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":200}}`), + "AI/D_AIGrowth.json": []byte(`{"Rows":[{"Name":"Mount_Bear","BaseMovementSpeed":200}]}`), } basePak := writeTestBasePak(t, baseTables) manifest := `{"name":"Bear Mount","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":235}]}]}` @@ -78,7 +78,7 @@ func TestCompile_AppliesDiffAndBundlesAssets(t *testing.T) { // data table (it has none). func TestCompile_SkipsEndOfModSentinelRow(t *testing.T) { baseTables := map[string][]byte{ - "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":200}}`), + "AI/D_AIGrowth.json": []byte(`{"Rows":[{"Name":"Mount_Bear","BaseMovementSpeed":200}]}`), } basePak := writeTestBasePak(t, baseTables) manifest := `{"name":"X","Rows":[` + @@ -130,7 +130,7 @@ func TestCompile_RowWithoutFileItems_Errors(t *testing.T) { // the pak's own bytes, not any other source. func TestCompile_PatchesTheBasePaksOwnTable(t *testing.T) { basePak := writeTestBasePak(t, map[string][]byte{ - "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":200,"OnlyInPak":true}}`), + "AI/D_AIGrowth.json": []byte(`{"Rows":[{"Name":"Mount_Bear","BaseMovementSpeed":200,"OnlyInPak":true}]}`), }) manifest := `{"name":"X","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":235}]}]}` exmodzPath := writeTestExmodzFile(t, manifest, nil) diff --git a/internal/source/icarus/exmod.go b/internal/source/icarus/exmod.go index be3c335..b40059f 100644 --- a/internal/source/icarus/exmod.go +++ b/internal/source/icarus/exmod.go @@ -22,10 +22,11 @@ type ExmodRow struct { FileItems []ExmodFileItem } -// ExmodFileItem overrides fields on the base row named Name. Fields holds -// every key from the source JSON except "Name" itself, generically — the -// real schema nests arbitrary game-data shapes here (see package doc -// comment), so this deliberately does not enumerate them. +// ExmodFileItem upserts fields on the base row named Name — patching it if +// it already exists, adding it as a new row otherwise (see ApplyRowPatch). +// Fields holds every key from the source JSON except "Name" itself, +// generically — the real schema nests arbitrary game-data shapes here (see +// package doc comment), so this deliberately does not enumerate them. type ExmodFileItem struct { Name string Fields map[string]any @@ -68,26 +69,77 @@ func ParseExmod(data []byte) (*ExmodDiff, error) { return diff, nil } -// ApplyRowPatch merges row's named-row field overrides into baseJSON (a base -// game data-table file keyed by row name, e.g. {"Mount_Bear": {...}, ...}) -// and returns the patched document. Fails loudly (no silent fallback, repo -// precedent #95) if a targeted row name doesn't exist in the base — that -// means either the base version is stale relative to the mod, or the exmod -// targets a file this function was called with by mistake. +// ApplyRowPatch applies row's File_Items to baseJSON, a real Icarus +// DataTable JSON export — the standard Unreal Engine shape +// {"RowStruct": "...", "Defaults": {...}, "Rows": [{"Name": "...", ...fields}, ...]}, +// confirmed against a real installed data.pak (task-7-report.md); not the +// flat {name: {fields}} map this function originally assumed, which never +// matched real game data and was only ever exercised against synthetic +// fixtures. +// +// Each File_Item is an upsert, not a strict patch: if its Name matches an +// existing entry in Rows, that row's fields are shallow-merged with the +// item's fields (item fields win, everything else on the row survives +// untouched); if no row has that Name, the item is appended to Rows +// verbatim as a brand-new row. This matches what real .EXMOD content +// actually does — most rows patch existing base stats, but a +// content-adding mod (e.g. a new mountable species) introduces rows the +// base game doesn't have yet, and erroring on that (the original +// patch-only design) made every such mod uncompilable. All other top-level +// keys on the base document (RowStruct, Defaults, and anything else) pass +// through re-serialization unchanged, since only doc["Rows"] is ever +// modified. Output is deterministic: encoding/json sorts map keys. func ApplyRowPatch(baseJSON []byte, row ExmodRow) ([]byte, error) { - var doc map[string]map[string]any + var doc map[string]any if err := json.Unmarshal(baseJSON, &doc); err != nil { return nil, fmt.Errorf("icarus: parsing base data table %s: %w", row.CurrentFile, err) } - for _, item := range row.FileItems { - target, ok := doc[item.Name] + rawRows, ok := doc["Rows"] + if !ok { + return nil, fmt.Errorf("icarus: base data table %s: no top-level %q array", row.CurrentFile, "Rows") + } + rowsSlice, ok := rawRows.([]any) + if !ok { + return nil, fmt.Errorf("icarus: base data table %s: %q is not an array", row.CurrentFile, "Rows") + } + rows := make([]map[string]any, len(rowsSlice)) + for i, r := range rowsSlice { + m, ok := r.(map[string]any) if !ok { - return nil, fmt.Errorf("icarus: %s: row %q not found in base data table", row.CurrentFile, item.Name) + return nil, fmt.Errorf("icarus: base data table %s: Rows[%d] is not an object", row.CurrentFile, i) + } + rows[i] = m + } + + byName := make(map[string]int, len(rows)) + for i, r := range rows { + if name, ok := r["Name"].(string); ok { + byName[name] = i + } + } + + for _, item := range row.FileItems { + if idx, ok := byName[item.Name]; ok { + target := rows[idx] + for k, v := range item.Fields { + target[k] = v + } + continue } + newRow := make(map[string]any, len(item.Fields)+1) + newRow["Name"] = item.Name for k, v := range item.Fields { - target[k] = v + newRow[k] = v } - doc[item.Name] = target + rows = append(rows, newRow) + byName[item.Name] = len(rows) - 1 } + + newRows := make([]any, len(rows)) + for i, r := range rows { + newRows[i] = r + } + doc["Rows"] = newRows + return json.Marshal(doc) } diff --git a/internal/source/icarus/exmod_test.go b/internal/source/icarus/exmod_test.go index 9b1d442..adff172 100644 --- a/internal/source/icarus/exmod_test.go +++ b/internal/source/icarus/exmod_test.go @@ -2,6 +2,7 @@ package icarus import ( "encoding/json" + "strings" "testing" ) @@ -39,11 +40,57 @@ func TestParseExmod(t *testing.T) { } } -func TestApplyRowPatch_OverwritesNamedRowFieldsOnly(t *testing.T) { - base := []byte(`{ - "Mount_Bear": {"BaseMovementSpeed": 200, "BaseSwimSpeed": 150, "Untouched": "keep-me"}, - "Other_Row": {"BaseMovementSpeed": 999} - }`) +// TestParseExmod_FileItemWithoutName_Errors pins the loud-error path for a +// File_Items entry with no Name — unchanged by the #175 ApplyRowPatch fix, +// but previously untested directly. +func TestParseExmod_FileItemWithoutName_Errors(t *testing.T) { + manifest := `{"name":"X","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"BaseMovementSpeed":235}]}]}` + _, err := ParseExmod([]byte(manifest)) + if err == nil { + t.Fatal("expected an error for a File_Item with no Name, got nil") + } + if !strings.Contains(err.Error(), "AI-D_AIGrowth.json") { + t.Errorf("error %q should name the offending row", err) + } +} + +// realBaseTable is a stand-in for a real Icarus DataTable JSON export — the +// standard Unreal Engine shape confirmed against a live data.pak +// (task-7-report.md): {"RowStruct", "Defaults", "Rows": [{"Name", ...}]}, +// not the flat {name: {fields}} map ApplyRowPatch originally (and +// incorrectly) assumed. +const realBaseTable = `{ + "RowStruct": "/Script/Icarus.AIGrowth", + "Defaults": {"Health": "None"}, + "Rows": [ + {"Name": "Mount_Bear", "BaseMovementSpeed": 200, "BaseSwimSpeed": 150, "Untouched": "keep-me"}, + {"Name": "Other_Row", "BaseMovementSpeed": 999} + ] +}` + +func decodeRows(t *testing.T, patched []byte) []map[string]any { + t.Helper() + var doc struct { + Rows []map[string]any `json:"Rows"` + } + if err := json.Unmarshal(patched, &doc); err != nil { + t.Fatalf("unmarshaling patched result: %v", err) + } + return doc.Rows +} + +func findRow(t *testing.T, rows []map[string]any, name string) map[string]any { + t.Helper() + for _, r := range rows { + if r["Name"] == name { + return r + } + } + t.Fatalf("no row named %q in %+v", name, rows) + return nil +} + +func TestApplyRowPatch_PatchesExistingRow(t *testing.T) { row := ExmodRow{ CurrentFile: "AI-D_AIGrowth.json", FileItems: []ExmodFileItem{ @@ -51,31 +98,151 @@ func TestApplyRowPatch_OverwritesNamedRowFieldsOnly(t *testing.T) { }, } - got, err := ApplyRowPatch(base, row) + got, err := ApplyRowPatch([]byte(realBaseTable), row) if err != nil { t.Fatalf("ApplyRowPatch: %v", err) } - var result map[string]map[string]any - if err := json.Unmarshal(got, &result); err != nil { - t.Fatalf("unmarshaling result: %v", err) + rows := decodeRows(t, got) + if len(rows) != 2 { + t.Fatalf("Rows = %d entries, want 2 (no rows added)", len(rows)) + } + mountBear := findRow(t, rows, "Mount_Bear") + if mountBear["BaseMovementSpeed"] != float64(235) { + t.Errorf("BaseMovementSpeed not patched: %v", mountBear["BaseMovementSpeed"]) + } + if mountBear["Untouched"] != "keep-me" { + t.Errorf("unrelated field was clobbered: %v", mountBear["Untouched"]) + } + other := findRow(t, rows, "Other_Row") + if other["BaseMovementSpeed"] != float64(999) { + t.Errorf("unrelated row was modified: %v", other) + } +} + +// TestApplyRowPatch_AddsNewRow pins the #175 fix: a File_Item whose Name has +// no match in the base table's Rows is appended as a brand-new row instead +// of erroring — real content-adding mods like Bear_Mount need this (see +// task-7-report.md's "Mount_Bear does not exist in the live install" +// finding). +func TestApplyRowPatch_AddsNewRow(t *testing.T) { + row := ExmodRow{ + CurrentFile: "AI-D_AIGrowth.json", + FileItems: []ExmodFileItem{ + {Name: "Juvenile_Bear", Fields: map[string]any{"BaseMovementSpeed": float64(144)}}, + }, + } + + got, err := ApplyRowPatch([]byte(realBaseTable), row) + if err != nil { + t.Fatalf("ApplyRowPatch: %v", err) + } + + rows := decodeRows(t, got) + if len(rows) != 3 { + t.Fatalf("Rows = %d entries, want 3 (2 original + 1 added)", len(rows)) + } + added := findRow(t, rows, "Juvenile_Bear") + if added["BaseMovementSpeed"] != float64(144) { + t.Errorf("added row BaseMovementSpeed = %v, want 144", added["BaseMovementSpeed"]) + } + if findRow(t, rows, "Mount_Bear")["BaseMovementSpeed"] != float64(200) { + t.Error("existing row was modified by an unrelated add") + } +} + +func TestApplyRowPatch_MixedPatchAndAddInOneTable(t *testing.T) { + row := ExmodRow{ + CurrentFile: "AI-D_AIGrowth.json", + FileItems: []ExmodFileItem{ + {Name: "Mount_Bear", Fields: map[string]any{"BaseMovementSpeed": float64(235)}}, + {Name: "Juvenile_Bear", Fields: map[string]any{"BaseMovementSpeed": float64(144)}}, + }, } - if result["Mount_Bear"]["BaseMovementSpeed"] != float64(235) { - t.Errorf("BaseMovementSpeed not patched: %v", result["Mount_Bear"]["BaseMovementSpeed"]) + + got, err := ApplyRowPatch([]byte(realBaseTable), row) + if err != nil { + t.Fatalf("ApplyRowPatch: %v", err) + } + + rows := decodeRows(t, got) + if len(rows) != 3 { + t.Fatalf("Rows = %d entries, want 3", len(rows)) } - if result["Mount_Bear"]["Untouched"] != "keep-me" { - t.Errorf("unrelated field was clobbered: %v", result["Mount_Bear"]["Untouched"]) + if findRow(t, rows, "Mount_Bear")["BaseMovementSpeed"] != float64(235) { + t.Error("existing row was not patched") } - if result["Other_Row"]["BaseMovementSpeed"] != float64(999) { - t.Errorf("unrelated row was modified: %v", result["Other_Row"]) + if findRow(t, rows, "Juvenile_Bear")["BaseMovementSpeed"] != float64(144) { + t.Error("new row was not added") } } -func TestApplyRowPatch_UnknownRowName_Errors(t *testing.T) { - base := []byte(`{"Mount_Bear": {}}`) - row := ExmodRow{FileItems: []ExmodFileItem{{Name: "Does_Not_Exist", Fields: map[string]any{"X": 1}}}} +func TestApplyRowPatch_PreservesDefaultsAndRowStruct(t *testing.T) { + row := ExmodRow{ + CurrentFile: "AI-D_AIGrowth.json", + FileItems: []ExmodFileItem{ + {Name: "Mount_Bear", Fields: map[string]any{"BaseMovementSpeed": float64(235)}}, + }, + } + + got, err := ApplyRowPatch([]byte(realBaseTable), row) + if err != nil { + t.Fatalf("ApplyRowPatch: %v", err) + } + + var doc map[string]any + if err := json.Unmarshal(got, &doc); err != nil { + t.Fatalf("unmarshaling result: %v", err) + } + if doc["RowStruct"] != "/Script/Icarus.AIGrowth" { + t.Errorf("RowStruct = %v, want preserved", doc["RowStruct"]) + } + defaults, ok := doc["Defaults"].(map[string]any) + if !ok || defaults["Health"] != "None" { + t.Errorf("Defaults = %v, want preserved", doc["Defaults"]) + } +} - if _, err := ApplyRowPatch(base, row); err == nil { - t.Error("expected error for unknown row name (no silent fallback), got nil") +func TestApplyRowPatch_Errors(t *testing.T) { + tests := []struct { + name string + base string + wantErr string + }{ + { + name: "unparseable base JSON", + base: `not json`, + wantErr: "AI-D_AIGrowth.json", + }, + { + name: "missing Rows key", + base: `{"RowStruct": "/Script/Icarus.AIGrowth", "Defaults": {}}`, + wantErr: "AI-D_AIGrowth.json", + }, + { + name: "Rows present but not an array", + base: `{"Rows": "not-an-array"}`, + wantErr: "AI-D_AIGrowth.json", + }, + { + name: "Rows entry not an object", + base: `{"Rows": ["not-an-object"]}`, + wantErr: "AI-D_AIGrowth.json", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + row := ExmodRow{ + CurrentFile: "AI-D_AIGrowth.json", + FileItems: []ExmodFileItem{{Name: "X", Fields: map[string]any{}}}, + } + _, err := ApplyRowPatch([]byte(tt.base), row) + if err == nil { + t.Fatalf("ApplyRowPatch(%s): expected an error, got nil", tt.name) + } + if !strings.Contains(err.Error(), tt.wantErr) { + t.Errorf("error %q should contain %q", err, tt.wantErr) + } + }) } } From 6915cf71fb221fcde2b4df065e2c76e71ddee803 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 12:54:10 -0400 Subject: [PATCH 30/96] fix: mount compiled Icarus paks at the game data loader path (#178) --- internal/source/icarus/compile.go | 39 +++++++++++++++++-- internal/source/icarus/compile_test.go | 49 +++++++++++++++++++++-- internal/unrealpak/pak.go | 16 ++++---- internal/unrealpak/reader.go | 42 ++++++++++++-------- internal/unrealpak/reader_test.go | 4 +- internal/unrealpak/writer.go | 38 ++++++++++++++---- internal/unrealpak/writer_test.go | 54 ++++++++++++++++++++++++++ internal/unrealpak/zlib_test.go | 4 +- 8 files changed, 204 insertions(+), 42 deletions(-) diff --git a/internal/source/icarus/compile.go b/internal/source/icarus/compile.go index 284965a..1e71c5e 100644 --- a/internal/source/icarus/compile.go +++ b/internal/source/icarus/compile.go @@ -23,6 +23,13 @@ import ( // There is no ctx parameter: every step is local file I/O over a ~2 MB pak, // with no network call and no long-running loop to cancel. The // source.Compiler interface still takes one, for implementations that need it. +// +// The compiled pak's mount point and table-entry paths (icarusContentMountPoint, +// icarusDataTablePrefix below) are Icarus-specific and deliberately live here +// rather than in internal/unrealpak, which stays game-agnostic — see +// unrealpak.Writer's WithMountPoint. They are not guessed: both were +// confirmed against two real, working prebuilt Icarus mod paks (#178; see +// docs/plans/2026-08-01-icarus-zlib-pivot.md's pak-divergence-report.md). func Compile(basePakPath, exmodzPath, outputPakPath string) (err error) { exmodzData, err := os.ReadFile(exmodzPath) if err != nil { @@ -39,7 +46,7 @@ func Compile(basePakPath, exmodzPath, outputPakPath string) (err error) { } defer base.Close() //nolint:errcheck - out, err := unrealpak.Create(outputPakPath) + out, err := unrealpak.Create(outputPakPath, unrealpak.WithMountPoint(icarusContentMountPoint)) if err != nil { return fmt.Errorf("icarus: creating %s: %w", outputPakPath, err) } @@ -84,8 +91,9 @@ func Compile(basePakPath, exmodzPath, outputPakPath string) (err error) { if err != nil { return err } - if err := out.AddFile(mountPath, patched); err != nil { - return fmt.Errorf("icarus: writing patched %s: %w", mountPath, err) + tablePath := icarusDataTablePrefix + mountPath + if err := out.AddFile(tablePath, patched); err != nil { + return fmt.Errorf("icarus: writing patched %s: %w", tablePath, err) } } @@ -94,6 +102,11 @@ func Compile(basePakPath, exmodzPath, outputPakPath string) (err error) { if err != nil { return err } + // No icarusDataTablePrefix here: bundled assets are content packages, + // not JSON data-table overrides. They need only icarusContentMountPoint + // (via the Writer's mount point) to land under Icarus/Content/ at + // their own namespace path — confirmed against a real asset-only + // prebuilt mod pak (TurretVariants; see pak-divergence-report.md). if err := out.AddFile(safePath, data); err != nil { return fmt.Errorf("icarus: writing bundled asset %s: %w", safePath, err) } @@ -105,6 +118,26 @@ func Compile(basePakPath, exmodzPath, outputPakPath string) (err error) { return nil } +// icarusContentMountPoint is the mount point a compiled _P.pak must declare +// for Icarus's own data-table mod loader to find it. "../../../" (this +// package's own default — see unrealpak.defaultMountPoint) resolves to the +// Steam install's outer game folder; real Icarus mods redescend from there +// with a literal "Icarus/Content/" — the game's UProject-root folder name is +// itself "Icarus" (confirmed both by real prebuilt mods' own mount strings +// and by data.pak's own on-disk nesting: .../Icarus/Icarus/Content/Data/data.pak). +// Confirmed against two independent real, working prebuilt mod paks +// (FloofLevelCap, Intreeg's 4XP) — see pak-divergence-report.md. +const icarusContentMountPoint = "../../../Icarus/Content/" + +// icarusDataTablePrefix is prepended to a patched base table's own +// mount-relative path (as read from data.pak, e.g. "Experience/D_ExperienceEvents.json") +// before it is written into the compiled pak. Real prebuilt mods land their +// table overrides at "Icarus/Content/data/" — confirmed +// byte-for-byte against FloofLevelCap.pak and Intreeg's 4XP.pak. It must NOT +// be applied to bundled assets (see the asset loop below): a single pak +// can't correctly address both classes with the same prefix. +const icarusDataTablePrefix = "data/" + // endOfModSentinel is a known .EXMOD ecosystem terminator row: real-world // manifests end their Rows array with {"CurrentFile":"EndOfMod"} and no // File_Items key at all. It targets no data table and carries no patch, so diff --git a/internal/source/icarus/compile_test.go b/internal/source/icarus/compile_test.go index 433b274..cbc15da 100644 --- a/internal/source/icarus/compile_test.go +++ b/internal/source/icarus/compile_test.go @@ -56,7 +56,7 @@ func TestCompile_AppliesDiffAndBundlesAssets(t *testing.T) { } defer r.Close() //nolint:errcheck - patched, err := r.ReadFile("AI/D_AIGrowth.json") + patched, err := r.ReadFile("data/AI/D_AIGrowth.json") if err != nil { t.Fatalf("ReadFile patched data table: %v", err) } @@ -73,6 +73,49 @@ func TestCompile_AppliesDiffAndBundlesAssets(t *testing.T) { } } +// TestCompile_MountsAtTheGamesDataLoaderPath pins #178: the compiled pak's +// MountPoint and patched-table entry paths must land where Icarus's own +// data-table mod loader actually looks (confirmed against two real prebuilt +// mod paks — see pak-divergence-report.md), not at the bare base-pak-relative +// path Compile previously wrote (which mounted fine but had no effect). +// Bundled assets keep their own unprefixed path; only table entries get the +// "data/" prefix — a single pak can't correctly address both classes with +// the same prefix, since assets need the mount point alone to reach +// Icarus/Content/, while tables additionally need "data/" beneath that. +func TestCompile_MountsAtTheGamesDataLoaderPath(t *testing.T) { + basePak := writeTestBasePak(t, map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Rows":[{"Name":"Mount_Bear","BaseMovementSpeed":200}]}`), + }) + manifest := `{"name":"X","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":235}]}]}` + exmodzPath := writeTestExmodzFile(t, manifest, map[string][]byte{ + "Bear_Mount/ASS/ITM/SK_ITM_Saddle_Bear.uasset": []byte("fake-asset"), + }) + outputPath := filepath.Join(t.TempDir(), "out.pak") + + if err := Compile(basePak, exmodzPath, outputPath); err != nil { + t.Fatalf("Compile: %v", err) + } + r, err := unrealpak.Open(outputPath) + if err != nil { + t.Fatalf("opening compiled output: %v", err) + } + defer r.Close() //nolint:errcheck + + const wantMount = "../../../Icarus/Content/" + if got := r.MountPoint(); got != wantMount { + t.Errorf("MountPoint = %q, want %q", got, wantMount) + } + if _, err := r.ReadFile("data/AI/D_AIGrowth.json"); err != nil { + t.Errorf("patched table must live at the data/-prefixed path: %v", err) + } + if _, err := r.ReadFile("AI/D_AIGrowth.json"); err == nil { + t.Error("patched table must NOT also exist at the unprefixed path") + } + if _, err := r.ReadFile("Bear_Mount/ASS/ITM/SK_ITM_Saddle_Bear.uasset"); err != nil { + t.Errorf("bundled asset must keep its own unprefixed path: %v", err) + } +} + // The real .EXMOD ecosystem terminates Rows with {"CurrentFile":"EndOfMod"} // and no File_Items key — Compile must skip it, not try to resolve it as a // data table (it has none). @@ -96,7 +139,7 @@ func TestCompile_SkipsEndOfModSentinelRow(t *testing.T) { t.Fatalf("opening compiled output: %v", err) } defer r.Close() //nolint:errcheck - patched, err := r.ReadFile("AI/D_AIGrowth.json") + patched, err := r.ReadFile("data/AI/D_AIGrowth.json") if err != nil { t.Fatalf("ReadFile patched data table: %v", err) } @@ -144,7 +187,7 @@ func TestCompile_PatchesTheBasePaksOwnTable(t *testing.T) { t.Fatalf("opening compiled output: %v", err) } defer r.Close() //nolint:errcheck - got, err := r.ReadFile("AI/D_AIGrowth.json") + got, err := r.ReadFile("data/AI/D_AIGrowth.json") if err != nil { t.Fatalf("ReadFile: %v", err) } diff --git a/internal/unrealpak/pak.go b/internal/unrealpak/pak.go index 61f8f70..11cb912 100644 --- a/internal/unrealpak/pak.go +++ b/internal/unrealpak/pak.go @@ -105,11 +105,13 @@ func hashPath(mountRelative string, seed uint64) uint64 { return h } -// defaultMountPoint is the mount point Writer stamps into the primary index. -// Icarus's own data.pak uses an absolute cook-machine path -// ("C:/BA/work/.../Temp/Data/"); "../../../" is the conventional relative form -// used by its pakchunks. Confirming which one a _P.pak needs to override -// Content/Data/data.pak in-game is a post-plan validation item. +// defaultMountPoint is the mount point Writer stamps into the primary index +// when the caller doesn't override it via WithMountPoint. It is the bare +// UE4 convention ("../../../", relative to /Binaries/Win64/"), +// deliberately game-agnostic — this package has no opinion on any specific +// game's directory layout. A caller writing paks for a real game (see +// internal/source/icarus's Writer usage, #178) supplies the mount point its +// own game's mod loader actually expects. const defaultMountPoint = "../../../" // writeFString writes a length-prefixed ANSI Unreal FString (length includes @@ -151,11 +153,11 @@ func storedEntryHeader(size int64, content []byte) []byte { // sub-index offsets it records point past its own end, but its length does not // depend on their values (they are fixed-width int64), so a first pass with // zero offsets measures it and a second pass writes the real ones. -func buildPrimaryIndex(numEntries int32, seed uint64, +func buildPrimaryIndex(mountPoint string, numEntries int32, seed uint64, phiOffset, phiSize int64, phiHash [20]byte, fdiOffset, fdiSize int64, fdiHash [20]byte, encoded []byte) []byte { var b bytes.Buffer - writeFString(&b, defaultMountPoint) + writeFString(&b, mountPoint) binary.Write(&b, binary.LittleEndian, numEntries) //nolint:errcheck binary.Write(&b, binary.LittleEndian, seed) //nolint:errcheck // PathHashSeed binary.Write(&b, binary.LittleEndian, int32(1)) //nolint:errcheck // bHasPathHashIndex diff --git a/internal/unrealpak/reader.go b/internal/unrealpak/reader.go index 8c60462..6ded9be 100644 --- a/internal/unrealpak/reader.go +++ b/internal/unrealpak/reader.go @@ -17,10 +17,11 @@ import ( // and Zlib-compressed entries are readable; any other compression method is a // loud ErrUnsupportedFormat. type Reader struct { - f *os.File - entries []readerEntry - fileSize int64 // total size of the underlying file, for validateAllocSize - methods [maxCompressionMethods]string // this pak's own CompressionMethods table + f *os.File + entries []readerEntry + fileSize int64 // total size of the underlying file, for validateAllocSize + methods [maxCompressionMethods]string // this pak's own CompressionMethods table + mountPoint string // this pak's own primary-index MountPoint (see Writer's WithMountPoint) } type readerEntry struct { @@ -64,13 +65,13 @@ func Open(path string) (*Reader, error) { return nil, fmt.Errorf("unrealpak: %s: primary index: %w", path, err) } - entries, err := parseIndex(f, indexBuf, fileSize) + mountPoint, entries, err := parseIndex(f, indexBuf, fileSize) if err != nil { f.Close() //nolint:errcheck return nil, fmt.Errorf("unrealpak: %s: parsing index: %w", path, err) } - return &Reader{f: f, entries: entries, fileSize: fileSize, methods: ft.methods}, nil + return &Reader{f: f, entries: entries, fileSize: fileSize, methods: ft.methods, mountPoint: mountPoint}, nil } // methodName resolves a 1-based CompressionMethodIndex against this pak's own @@ -128,6 +129,13 @@ func readRegion(r io.ReaderAt, offset, size, fileSize int64, want [20]byte) ([]b // Close releases the underlying file handle. func (r *Reader) Close() error { return r.f.Close() } +// MountPoint returns this pak's primary-index MountPoint string — where the +// engine roots Files' mount-relative paths once the pak is mounted. Real +// paks vary this: Icarus's own data.pak declares an absolute cook-machine +// path, while mod paks conventionally declare a relative "../../../..." +// form (see Writer's WithMountPoint, #178). +func (r *Reader) MountPoint() string { return r.mountPoint } + // Files returns every file this pak's index describes. func (r *Reader) Files() []FileEntry { out := make([]FileEntry, len(r.entries)) @@ -359,50 +367,50 @@ func readFooter(r io.ReaderAt, fileSize int64) (footer, error) { // index (hash -> record offset) and a full directory index // (directory -> file -> record offset). Enumeration uses the directory index, // which is the only one that carries real path strings. -func parseIndex(f io.ReaderAt, index []byte, fileSize int64) ([]readerEntry, error) { +func parseIndex(f io.ReaderAt, index []byte, fileSize int64) (string, []readerEntry, error) { c := &cursor{b: index} - c.fstring() // MountPoint: recorded for the engine's benefit, unused here + mountPoint := c.fstring() numEntries := c.i32() seed := c.u64() _ = seed // only the writer needs the seed; enumeration goes via the directory index pathHash, err := readSubIndexRef(c, "path hash index") if err != nil { - return nil, err + return "", nil, err } fullDir, err := readSubIndexRef(c, "full directory index") if err != nil { - return nil, err + return "", nil, err } encoded := c.bytes(int(c.i32())) // EncodedPakEntriesSize, then the blob if nonEncoded := c.i32(); nonEncoded != 0 { - return nil, fmt.Errorf("%w: %d non-encoded index entries", ErrUnsupportedFormat, nonEncoded) + return "", nil, fmt.Errorf("%w: %d non-encoded index entries", ErrUnsupportedFormat, nonEncoded) } if c.err != nil { - return nil, fmt.Errorf("primary index: %w", c.err) + return "", nil, fmt.Errorf("primary index: %w", c.err) } // Verify the path-hash index's hash even though enumeration does not use // it: it is part of the format's integrity chain, and a pak whose // sub-index hashes don't hold is not one to trust. if _, err := readRegion(f, pathHash.offset, pathHash.size, fileSize, pathHash.hash); err != nil { - return nil, fmt.Errorf("path hash index: %w", err) + return "", nil, fmt.Errorf("path hash index: %w", err) } dirBuf, err := readRegion(f, fullDir.offset, fullDir.size, fileSize, fullDir.hash) if err != nil { - return nil, fmt.Errorf("full directory index: %w", err) + return "", nil, fmt.Errorf("full directory index: %w", err) } entries, err := parseDirectoryIndex(dirBuf, encoded) if err != nil { - return nil, err + return "", nil, err } if int32(len(entries)) != numEntries { - return nil, fmt.Errorf("directory index lists %d files, index header says %d", + return "", nil, fmt.Errorf("directory index lists %d files, index header says %d", len(entries), numEntries) } sort.Slice(entries, func(i, j int) bool { return entries[i].Path < entries[j].Path }) - return entries, nil + return mountPoint, entries, nil } type subIndexRef struct { diff --git a/internal/unrealpak/reader_test.go b/internal/unrealpak/reader_test.go index 0112b43..064fc92 100644 --- a/internal/unrealpak/reader_test.go +++ b/internal/unrealpak/reader_test.go @@ -85,10 +85,10 @@ func buildFixturePak(mountPath string, content []byte, method int32) []byte { fdiHash := sha1.Sum(fdi.Bytes()) //nolint:gosec indexOffset := int64(data.Len()) - sizing := buildPrimaryIndex(1, fixtureSeed, 0, 0, phiHash, 0, 0, fdiHash, encoded.Bytes()) + sizing := buildPrimaryIndex(defaultMountPoint, 1, fixtureSeed, 0, 0, phiHash, 0, 0, fdiHash, encoded.Bytes()) phiOffset := indexOffset + int64(len(sizing)) fdiOffset := phiOffset + int64(phi.Len()) - index := buildPrimaryIndex(1, fixtureSeed, + index := buildPrimaryIndex(defaultMountPoint, 1, fixtureSeed, phiOffset, int64(phi.Len()), phiHash, fdiOffset, int64(fdi.Len()), fdiHash, encoded.Bytes()) indexHash := sha1.Sum(index) //nolint:gosec diff --git a/internal/unrealpak/writer.go b/internal/unrealpak/writer.go index 8489bf9..c14ad9a 100644 --- a/internal/unrealpak/writer.go +++ b/internal/unrealpak/writer.go @@ -28,10 +28,11 @@ const writerSeed uint64 = 0x9E3779B97F4A7C15 // round-trip test able to assert on bytes and keeps compiled paks stable across // recompiles. type Writer struct { - f *os.File - closed bool - files []writerFile - seen map[string]bool + f *os.File + closed bool + files []writerFile + seen map[string]bool + mountPoint string } type writerFile struct { @@ -39,13 +40,34 @@ type writerFile struct { data []byte } +// Option configures a Writer at construction time (see Create). The set is +// deliberately tiny and additive: new options can be introduced without +// breaking existing Create(path) call sites, since opts is variadic. +type Option func(*Writer) + +// WithMountPoint overrides the mount point Writer stamps into the primary +// index (see defaultMountPoint). This package stays game-agnostic — it has +// no built-in notion of any specific game's directory layout — so a caller +// that knows what its target game's mod loader expects (e.g. +// internal/source/icarus, #178) supplies it here rather than this package +// guessing or hard-coding one game's convention. +func WithMountPoint(mountPoint string) Option { + return func(w *Writer) { w.mountPoint = mountPoint } +} + // Create opens path for writing. Call AddFile for each entry, then Close. -func Create(path string) (*Writer, error) { +// With no options, the written pak uses defaultMountPoint, matching every +// existing caller's prior behavior exactly. +func Create(path string, opts ...Option) (*Writer, error) { f, err := os.Create(path) if err != nil { return nil, fmt.Errorf("unrealpak: creating %s: %w", path, err) } - return &Writer{f: f, seen: make(map[string]bool)}, nil + w := &Writer{f: f, seen: make(map[string]bool), mountPoint: defaultMountPoint} + for _, opt := range opts { + opt(w) + } + return w, nil } // AddFile records one entry. Nothing reaches disk until Close. @@ -157,10 +179,10 @@ func (w *Writer) Close() error { fdiHash := sha1.Sum(fdi.Bytes()) //nolint:gosec count := int32(len(w.files)) indexOffset := int64(data.Len()) - sizing := buildPrimaryIndex(count, writerSeed, 0, 0, phiHash, 0, 0, fdiHash, encoded.Bytes()) + sizing := buildPrimaryIndex(w.mountPoint, count, writerSeed, 0, 0, phiHash, 0, 0, fdiHash, encoded.Bytes()) phiOffset := indexOffset + int64(len(sizing)) fdiOffset := phiOffset + int64(phi.Len()) - index := buildPrimaryIndex(count, writerSeed, + index := buildPrimaryIndex(w.mountPoint, count, writerSeed, phiOffset, int64(phi.Len()), phiHash, fdiOffset, int64(fdi.Len()), fdiHash, encoded.Bytes()) indexHash := sha1.Sum(index) //nolint:gosec diff --git a/internal/unrealpak/writer_test.go b/internal/unrealpak/writer_test.go index 6edfbd0..fa2f13c 100644 --- a/internal/unrealpak/writer_test.go +++ b/internal/unrealpak/writer_test.go @@ -100,6 +100,60 @@ func TestWriter_AddFile_AfterClose_Errors(t *testing.T) { } } +// TestWriter_Create_DefaultMountPoint pins that Create with no options +// preserves every prior caller's behavior exactly: the written pak's +// MountPoint is the package's own defaultMountPoint. +func TestWriter_Create_DefaultMountPoint(t *testing.T) { + path := filepath.Join(t.TempDir(), "out.pak") + w, err := Create(path) + if err != nil { + t.Fatalf("Create: %v", err) + } + if err := w.AddFile("x.json", []byte("{}")); err != nil { + t.Fatalf("AddFile: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + r, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() //nolint:errcheck + if got := r.MountPoint(); got != defaultMountPoint { + t.Errorf("MountPoint = %q, want default %q", got, defaultMountPoint) + } +} + +// TestWriter_Create_WithMountPoint pins that WithMountPoint overrides the +// stamped mount point — this package stays game-agnostic (#178), so a +// caller like internal/source/icarus supplies its own game's mount point +// through this seam rather than this package hard-coding one. +func TestWriter_Create_WithMountPoint(t *testing.T) { + path := filepath.Join(t.TempDir(), "out.pak") + const custom = "../../../Icarus/Content/" + w, err := Create(path, WithMountPoint(custom)) + if err != nil { + t.Fatalf("Create: %v", err) + } + if err := w.AddFile("data/x.json", []byte("{}")); err != nil { + t.Fatalf("AddFile: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + r, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() //nolint:errcheck + if got := r.MountPoint(); got != custom { + t.Errorf("MountPoint = %q, want %q", got, custom) + } +} + // checkEncodedLocationFits is tested directly on the boundary rather than by // constructing a >2 GiB encoded-index fixture, which would be impractically // slow and memory-hungry for a unit test. diff --git a/internal/unrealpak/zlib_test.go b/internal/unrealpak/zlib_test.go index 620cc11..86a69a2 100644 --- a/internal/unrealpak/zlib_test.go +++ b/internal/unrealpak/zlib_test.go @@ -114,10 +114,10 @@ func writeMethodPak(t *testing.T, methods []string, fixtures []zlibFixture) stri fdiHash := sha1.Sum(fdi.Bytes()) //nolint:gosec count := int32(len(fixtures)) indexOffset := int64(data.Len()) - sizing := buildPrimaryIndex(count, seed, 0, 0, phiHash, 0, 0, fdiHash, encoded.Bytes()) + sizing := buildPrimaryIndex(defaultMountPoint, count, seed, 0, 0, phiHash, 0, 0, fdiHash, encoded.Bytes()) phiOffset := indexOffset + int64(len(sizing)) fdiOffset := phiOffset + int64(phi.Len()) - index := buildPrimaryIndex(count, seed, phiOffset, int64(phi.Len()), phiHash, + index := buildPrimaryIndex(defaultMountPoint, count, seed, phiOffset, int64(phi.Len()), phiHash, fdiOffset, int64(fdi.Len()), fdiHash, encoded.Bytes()) indexHash := sha1.Sum(index) //nolint:gosec From 0a420568c714919da2f274acee502060423c6080 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 13:14:33 -0400 Subject: [PATCH 31/96] feat: auto-detect Icarus in lmm game detect (#177) --- CHANGELOG.md | 1 + README.md | 2 +- cmd/lmm/game.go | 33 ++++-- cmd/lmm/game_detect_test.go | 112 ++++++++++++++++++++ docs/configuration.md | 9 +- internal/source/steam/data/steam-games.yaml | 11 ++ internal/source/steam/games.go | 33 ++++-- internal/source/steam/games_test.go | 48 +++++++++ internal/source/steam/steam.go | 16 +-- internal/source/steam/steam_test.go | 32 ++++++ 10 files changed, 274 insertions(+), 23 deletions(-) create mode 100644 cmd/lmm/game_detect_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index d64762d..e53ce68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - **Icarus built-in mod source** (`internal/source/icarus`): a public, unauthenticated Firestore-backed catalog (Project Daedalus) — `lmm search`/`install`/`update` work against it like NexusMods/CurseForge. A `.exmodz` mod file now compiles into a deployable `_P.pak` at download time via a new, game-agnostic `internal/unrealpak` PAK reader/writer and the new `deploy_mode: compile` game setting; a plain `.pak` file from the same catalog is unaffected and deploys through the existing extract/copy pipeline unchanged. Base data tables are read directly from the installed game's own `data.pak`, so a compile always matches the installed game version and works entirely offline; `internal/unrealpak` reads both the stored and the Zlib-compressed entries that pak contains, using only the standard library (#136, #175) +- `lmm game detect` now recognizes Icarus (Steam App ID `1149460`) and generates a complete `games.yaml` entry for it (`deploy_mode: compile`, `sources: {icarus: icarus}`) — no more hand-editing `games.yaml` to get started. The known-games schema (`steam-games.yaml`, built-in or your own override) gained two optional fields, `deploy_mode` and `sources`, generalizing detection beyond NexusMods-only games; every existing entry is unaffected (#177) ## [1.27.1] - 2026-07-30 diff --git a/README.md b/README.md index 2717a8a..ecae985 100644 --- a/README.md +++ b/README.md @@ -431,7 +431,7 @@ games: icarus: "icarus" ``` -Steam auto-detection (`lmm game detect`) does not yet know about Icarus (App ID `1149460`, confirmed during the research spike) — add this entry to `games.yaml` by hand for now; auto-detection is a separate, smaller follow-up not covered by this plan. +Steam auto-detection (`lmm game detect`) knows about Icarus (App ID `1149460`) and generates exactly this block for you, `install_path`/`mod_path` filled in from your actual Steam library — the YAML above is kept here as reference for what gets written, not something you need to type by hand. ### Deployment Methods diff --git a/cmd/lmm/game.go b/cmd/lmm/game.go index b6171f4..a417dcb 100644 --- a/cmd/lmm/game.go +++ b/cmd/lmm/game.go @@ -225,14 +225,7 @@ func runGameDetect(cmd *cobra.Command, args []string) error { } for _, n := range indices { g := games[n-1] - game := &domain.Game{ - ID: g.Slug, - Name: g.Name, - InstallPath: g.InstallPath, - ModPath: g.ModPath, - SourceIDs: map[string]string{"nexusmods": g.NexusID}, - LinkMethod: domain.LinkSymlink, - } + game := gameFromDetected(g) if err := config.SaveGame(svcCfg.ConfigDir, game); err != nil { return fmt.Errorf("saving game %s: %w", g.Slug, err) } @@ -251,3 +244,27 @@ func runGameDetect(cmd *cobra.Command, args []string) error { } return nil } + +// gameFromDetected converts one steam.DetectedGame into the domain.Game +// runGameDetect saves. g.Sources, when the known-games entry supplied one +// (#177: games with a non-NexusMods or multi-source setup, e.g. Icarus), +// wins outright; otherwise this derives the single-entry {nexusmods: +// g.NexusID} map every detected game produced before Sources existed, so +// every pre-#177 known game generates byte-for-byte the same games.yaml +// block it always has. g.DeployMode goes through domain.ParseDeployMode, +// which already treats "" as DeployExtract (today's default). +func gameFromDetected(g steam.DetectedGame) *domain.Game { + sources := g.Sources + if sources == nil { + sources = map[string]string{"nexusmods": g.NexusID} + } + return &domain.Game{ + ID: g.Slug, + Name: g.Name, + InstallPath: g.InstallPath, + ModPath: g.ModPath, + SourceIDs: sources, + LinkMethod: domain.LinkSymlink, + DeployMode: domain.ParseDeployMode(g.DeployMode), + } +} diff --git a/cmd/lmm/game_detect_test.go b/cmd/lmm/game_detect_test.go new file mode 100644 index 0000000..7214c6c --- /dev/null +++ b/cmd/lmm/game_detect_test.go @@ -0,0 +1,112 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/source/steam" + "github.com/DonovanMods/linux-mod-manager/internal/storage/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +// TestGameFromDetected pins #177's conversion from a steam.DetectedGame to +// the domain.Game runGameDetect saves — table-driven so the untouched-shape +// case (every existing known game: NexusID only, no DeployMode/Sources) and +// the new Icarus-shape case (multi-key Sources map + compile DeployMode) are +// pinned side by side. +func TestGameFromDetected(t *testing.T) { + tests := []struct { + name string + in steam.DetectedGame + want *domain.Game + }{ + { + name: "NexusMods-only game keeps today's exact shape", + in: steam.DetectedGame{ + Slug: "skyrim-se", + Name: "Skyrim Special Edition", + InstallPath: "/games/skyrim", + ModPath: "/games/skyrim/Data", + NexusID: "skyrimspecialedition", + }, + want: &domain.Game{ + ID: "skyrim-se", + Name: "Skyrim Special Edition", + InstallPath: "/games/skyrim", + ModPath: "/games/skyrim/Data", + SourceIDs: map[string]string{"nexusmods": "skyrimspecialedition"}, + LinkMethod: domain.LinkSymlink, + DeployMode: domain.DeployExtract, + }, + }, + { + name: "Icarus: explicit Sources map + compile DeployMode", + in: steam.DetectedGame{ + Slug: "icarus", + Name: "Icarus", + InstallPath: "/games/Icarus", + ModPath: "/games/Icarus/Icarus/Content/Paks/mods", + DeployMode: "compile", + Sources: map[string]string{"icarus": "icarus"}, + }, + want: &domain.Game{ + ID: "icarus", + Name: "Icarus", + InstallPath: "/games/Icarus", + ModPath: "/games/Icarus/Icarus/Content/Paks/mods", + SourceIDs: map[string]string{"icarus": "icarus"}, + LinkMethod: domain.LinkSymlink, + DeployMode: domain.DeployCompile, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := gameFromDetected(tt.in) + assert.Equal(t, tt.want, got) + }) + } +} + +// TestGameFromDetected_Icarus_ProducesReadmeEquivalentGamesYAML proves the +// #177 acceptance criterion directly: saving a detected Icarus produces a +// games.yaml block equivalent to the README's hand-written example (the one +// users no longer need to type themselves). +func TestGameFromDetected_Icarus_ProducesReadmeEquivalentGamesYAML(t *testing.T) { + dir := t.TempDir() + detected := steam.DetectedGame{ + Slug: "icarus", + Name: "Icarus", + InstallPath: "/path/to/Steam/steamapps/common/Icarus", + ModPath: "/path/to/Steam/steamapps/common/Icarus/Icarus/Content/Paks/mods", + DeployMode: "compile", + Sources: map[string]string{"icarus": "icarus"}, + } + require.NoError(t, config.SaveGame(dir, gameFromDetected(detected))) + + data, err := os.ReadFile(filepath.Join(dir, "games.yaml")) + require.NoError(t, err) + + var parsed struct { + Games map[string]struct { + Name string `yaml:"name"` + InstallPath string `yaml:"install_path"` + ModPath string `yaml:"mod_path"` + Sources map[string]string `yaml:"sources"` + DeployMode string `yaml:"deploy_mode"` + } `yaml:"games"` + } + require.NoError(t, yaml.Unmarshal(data, &parsed)) + + got, ok := parsed.Games["icarus"] + require.True(t, ok, "games.yaml should have an 'icarus' entry") + assert.Equal(t, "Icarus", got.Name) + assert.Equal(t, "/path/to/Steam/steamapps/common/Icarus", got.InstallPath) + assert.Equal(t, "/path/to/Steam/steamapps/common/Icarus/Icarus/Content/Paks/mods", got.ModPath) + assert.Equal(t, "compile", got.DeployMode) + assert.Equal(t, map[string]string{"icarus": "icarus"}, got.Sources) +} diff --git a/docs/configuration.md b/docs/configuration.md index b40cb57..0deebe2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -105,7 +105,7 @@ Used by `lmm game detect` to know which Steam games are moddable. The app ships **`~/.config/lmm/steam-games.yaml`** -Format: Steam App ID (string) as key, then `slug`, `name`, `nexus_id`, `mod_path` (relative to game install, empty for game root). Example: +Format: Steam App ID (string) as key, then `slug`, `name`, `mod_path` (relative to game install, empty for game root), optional `nexus_id` (omit for a game with no NexusMods presence), and two more optional fields, `deploy_mode` and `sources`, that pass straight through to the generated `games.yaml` entry's own `deploy_mode`/`sources` (omit both for the default `{nexusmods: }` sources map and `extract` deploy mode every entry got before these existed). Example: ```yaml "489830": @@ -118,6 +118,13 @@ Format: Steam App ID (string) as key, then `slug`, `name`, `nexus_id`, `mod_path name: My Game nexus_id: mygame mod_path: "" +"7654321": + slug: my-compile-game + name: My Compile-Mode Game + mod_path: Mods + deploy_mode: compile + sources: + mysource: my-compile-game ``` Entries here are merged with the built-in list (overrides win). No rebuild needed to support more games. diff --git a/internal/source/steam/data/steam-games.yaml b/internal/source/steam/data/steam-games.yaml index 1ae2446..edc69fb 100644 --- a/internal/source/steam/data/steam-games.yaml +++ b/internal/source/steam/data/steam-games.yaml @@ -1,6 +1,10 @@ # Steam App ID -> lmm game info for "lmm game detect". # Add or override entries in ~/.config/lmm/steam-games.yaml without rebuilding. # Keys are Steam App IDs (strings). mod_path is relative to game install (empty = game root). +# nexus_id is optional (omit for games with no NexusMods presence, e.g. Icarus). +# deploy_mode and sources are optional, generated-games.yaml passthroughs +# (games.yaml's deploy_mode/sources: see docs/configuration.md); omit both to +# get today's default shape ({nexusmods: }, deploy_mode extract). "489830": slug: skyrim-se name: Skyrim Special Edition @@ -61,3 +65,10 @@ name: "Call of Duty: Black Ops 6" nexus_id: callofdutyblackops6 mod_path: "" +"1149460": + slug: icarus + name: Icarus + mod_path: Icarus/Content/Paks/mods + deploy_mode: compile + sources: + icarus: icarus diff --git a/internal/source/steam/games.go b/internal/source/steam/games.go index cf6cc61..cba6ad3 100644 --- a/internal/source/steam/games.go +++ b/internal/source/steam/games.go @@ -18,16 +18,29 @@ const defaultSteamGamesPath = "data/steam-games.yaml" type GameInfo struct { Slug string // lmm game ID, e.g. "skyrim-se" Name string // Display name, e.g. "Skyrim Special Edition" - NexusID string // NexusMods game domain ID, e.g. "skyrimspecialedition" + NexusID string // NexusMods game domain ID, e.g. "skyrimspecialedition". Optional: absent for games with no NexusMods presence (e.g. Icarus). ModPath string // Relative path from game install to mod directory, e.g. "Data" + // DeployMode is games.yaml's deploy_mode string ("extract"/"copy"/"compile"), + // passed through as-is for domain.ParseDeployMode to interpret. Optional: + // "" means the game uses the default (extract), exactly as every entry + // behaved before this field existed (#177). + DeployMode string + // Sources is a full source-id -> per-source game-id map (games.yaml's + // "sources:" block), for games whose primary source isn't NexusMods (or + // that need more than one source, e.g. #177's Icarus: {"icarus": "icarus"}). + // Optional: nil means "derive {nexusmods: NexusID}", exactly as every + // entry behaved before this field existed. + Sources map[string]string } // steamGamesYAML is the on-disk format: Steam App ID -> game entry. type steamGamesYAML map[string]struct { - Slug string `yaml:"slug"` - Name string `yaml:"name"` - NexusID string `yaml:"nexus_id"` - ModPath string `yaml:"mod_path"` + Slug string `yaml:"slug"` + Name string `yaml:"name"` + NexusID string `yaml:"nexus_id,omitempty"` + ModPath string `yaml:"mod_path"` + DeployMode string `yaml:"deploy_mode,omitempty"` + Sources map[string]string `yaml:"sources,omitempty"` } // LoadKnownGames returns the known Steam App ID -> GameInfo map. It loads the @@ -44,7 +57,10 @@ func LoadKnownGames(configDir string) (map[string]GameInfo, error) { } out := make(map[string]GameInfo) for appID, e := range y { - out[appID] = GameInfo{Slug: e.Slug, Name: e.Name, NexusID: e.NexusID, ModPath: e.ModPath} + out[appID] = GameInfo{ + Slug: e.Slug, Name: e.Name, NexusID: e.NexusID, ModPath: e.ModPath, + DeployMode: e.DeployMode, Sources: e.Sources, + } } overridePath := filepath.Join(configDir, "steam-games.yaml") @@ -60,7 +76,10 @@ func LoadKnownGames(configDir string) (map[string]GameInfo, error) { return nil, fmt.Errorf("parsing %s: %w", overridePath, err) } for appID, e := range override { - out[appID] = GameInfo{Slug: e.Slug, Name: e.Name, NexusID: e.NexusID, ModPath: e.ModPath} + out[appID] = GameInfo{ + Slug: e.Slug, Name: e.Name, NexusID: e.NexusID, ModPath: e.ModPath, + DeployMode: e.DeployMode, Sources: e.Sources, + } } return out, nil } diff --git a/internal/source/steam/games_test.go b/internal/source/steam/games_test.go index 27bb293..778abed 100644 --- a/internal/source/steam/games_test.go +++ b/internal/source/steam/games_test.go @@ -43,7 +43,55 @@ func TestLoadKnownGames_OverrideFile(t *testing.T) { assert.Equal(t, "Test Game", info.Name) assert.Equal(t, "testgame", info.NexusID) assert.Equal(t, "Mods", info.ModPath) + // Optional fields absent from this override: must be the zero value, not + // inherited or defaulted from anywhere. + assert.Equal(t, "", info.DeployMode) + assert.Nil(t, info.Sources) // Embedded default still present _, ok = games["489830"] require.True(t, ok) } + +// TestLoadKnownGames_IcarusEntry pins the #177 known-games entry: Icarus has +// no NexusMods presence (nexus_id absent, unlike every other embedded game), +// and needs the two new optional fields (deploy_mode, sources) that #175's +// compile pipeline and games.yaml schema already support. +func TestLoadKnownGames_IcarusEntry(t *testing.T) { + games, err := LoadKnownGames(t.TempDir()) + require.NoError(t, err) + info, ok := games["1149460"] + require.True(t, ok) + assert.Equal(t, "icarus", info.Slug) + assert.Equal(t, "Icarus", info.Name) + assert.Equal(t, "", info.NexusID) + assert.Equal(t, "Icarus/Content/Paks/mods", info.ModPath) + assert.Equal(t, "compile", info.DeployMode) + assert.Equal(t, map[string]string{"icarus": "icarus"}, info.Sources) +} + +// TestLoadKnownGames_OverrideFile_DeployModeAndSources pins that the two new +// optional fields round-trip through a user's ~/.config/lmm/steam-games.yaml +// override exactly like every existing field already does — the schema +// extension isn't Icarus-only wiring, any override entry can use it. +func TestLoadKnownGames_OverrideFile_DeployModeAndSources(t *testing.T) { + dir := t.TempDir() + overridePath := filepath.Join(dir, "steam-games.yaml") + overrideYAML := ` +"888888": + slug: custom-compile-game + name: Custom Compile Game + mod_path: Mods + deploy_mode: compile + sources: + customsrc: customsrc-id +` + require.NoError(t, os.WriteFile(overridePath, []byte(overrideYAML), 0644)) + + games, err := LoadKnownGames(dir) + require.NoError(t, err) + info, ok := games["888888"] + require.True(t, ok) + assert.Equal(t, "compile", info.DeployMode) + assert.Equal(t, map[string]string{"customsrc": "customsrc-id"}, info.Sources) + assert.Equal(t, "", info.NexusID) // optional, absent in this override too +} diff --git a/internal/source/steam/steam.go b/internal/source/steam/steam.go index 1df6ac1..852fc7b 100644 --- a/internal/source/steam/steam.go +++ b/internal/source/steam/steam.go @@ -9,12 +9,14 @@ import ( // DetectedGame is a Steam game found on disk that lmm knows how to configure. type DetectedGame struct { - SteamAppID string // Steam App ID - Slug string // lmm game ID (from known games list) - Name string // Display name - InstallPath string // Absolute path to game install (e.g. .../common/Skyrim Special Edition) - ModPath string // Absolute path to mod directory (InstallPath + ModPath relative) - NexusID string // NexusMods game domain ID + SteamAppID string // Steam App ID + Slug string // lmm game ID (from known games list) + Name string // Display name + InstallPath string // Absolute path to game install (e.g. .../common/Skyrim Special Edition) + ModPath string // Absolute path to mod directory (InstallPath + ModPath relative) + NexusID string // NexusMods game domain ID. Optional: "" for games with no NexusMods presence (#177). + DeployMode string // games.yaml's deploy_mode string, passed through from GameInfo.DeployMode. Optional: "" means the default (extract). + Sources map[string]string // games.yaml's sources map, passed through from GameInfo.Sources. Optional: nil means "derive {nexusmods: NexusID}". } // FindSteamRoots returns candidate Steam installation roots in search order. @@ -141,6 +143,8 @@ func DetectGames(configDir string) (games []DetectedGame, warnings []string, err InstallPath: installPath, ModPath: modPath, NexusID: info.NexusID, + DeployMode: info.DeployMode, + Sources: info.Sources, }) } } diff --git a/internal/source/steam/steam_test.go b/internal/source/steam/steam_test.go index 5ed779c..fc8aaee 100644 --- a/internal/source/steam/steam_test.go +++ b/internal/source/steam/steam_test.go @@ -260,3 +260,35 @@ func TestDetectGames_DedupsSameGameAcrossLibraries(t *testing.T) { assert.Empty(t, warnings) require.Len(t, games, 1, "the same slug found in a second library must be deduped") } + +// TestDetectGames_IcarusEntry_IncludesDeployModeAndSources pins #177: a +// detected Icarus install carries the new DeployMode/Sources fields through +// from the known-games entry, and its ModPath is joined exactly like every +// other detected game's (installPath + the known entry's relative mod_path, +// here "Icarus/Content/Paks/mods" — matching the README's hand-written +// example, which this detection path now generates instead of requiring by +// hand). +func TestDetectGames_IcarusEntry_IncludesDeployModeAndSources(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("STEAM_ROOT", "") + + steamapps := filepath.Join(home, ".steam", "steam", "steamapps") + installDir := filepath.Join(steamapps, "common", "Icarus") + require.NoError(t, os.MkdirAll(installDir, 0755)) + writeAppManifest(t, steamapps, "1149460", "Icarus") + + games, warnings, err := DetectGames(t.TempDir()) + require.NoError(t, err) + assert.Empty(t, warnings) + require.Len(t, games, 1) + g := games[0] + assert.Equal(t, "1149460", g.SteamAppID) + assert.Equal(t, "icarus", g.Slug) + assert.Equal(t, "Icarus", g.Name) + assert.Equal(t, installDir, g.InstallPath) + assert.Equal(t, filepath.Join(installDir, "Icarus", "Content", "Paks", "mods"), g.ModPath) + assert.Equal(t, "", g.NexusID) + assert.Equal(t, "compile", g.DeployMode) + assert.Equal(t, map[string]string{"icarus": "icarus"}, g.Sources) +} From 6d43df2508f125f28f891e25cca9ebaec4d78cdb Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 13:25:20 -0400 Subject: [PATCH 32/96] docs: align auto-detect docs/test naming with value-equivalence (review) (#180) --- README.md | 4 ++-- cmd/lmm/game_detect_test.go | 10 ++++++---- internal/source/steam/data/steam-games.yaml | 6 +++--- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index ecae985..ecfb91a 100644 --- a/README.md +++ b/README.md @@ -426,12 +426,12 @@ games: name: "Icarus" install_path: "/path/to/Steam/steamapps/common/Icarus" mod_path: "/path/to/Steam/steamapps/common/Icarus/Icarus/Content/Paks/mods" - deploy_mode: compile sources: icarus: "icarus" + deploy_mode: compile ``` -Steam auto-detection (`lmm game detect`) knows about Icarus (App ID `1149460`) and generates exactly this block for you, `install_path`/`mod_path` filled in from your actual Steam library — the YAML above is kept here as reference for what gets written, not something you need to type by hand. +Steam auto-detection (`lmm game detect`) knows about Icarus (App ID `1149460`) and generates an equivalent entry for you, `install_path`/`mod_path` filled in from your actual Steam library — the YAML above is kept here as reference for what gets written, not something you need to type by hand. ### Deployment Methods diff --git a/cmd/lmm/game_detect_test.go b/cmd/lmm/game_detect_test.go index 7214c6c..513929e 100644 --- a/cmd/lmm/game_detect_test.go +++ b/cmd/lmm/game_detect_test.go @@ -72,11 +72,13 @@ func TestGameFromDetected(t *testing.T) { } } -// TestGameFromDetected_Icarus_ProducesReadmeEquivalentGamesYAML proves the +// TestGameFromDetected_Icarus_ProducesReadmeEquivalentValues proves the // #177 acceptance criterion directly: saving a detected Icarus produces a -// games.yaml block equivalent to the README's hand-written example (the one -// users no longer need to type themselves). -func TestGameFromDetected_Icarus_ProducesReadmeEquivalentGamesYAML(t *testing.T) { +// games.yaml entry whose values match the README's hand-written example +// (the one users no longer need to type themselves) — this asserts the +// parsed field values, not the YAML's byte-for-byte formatting, since only +// the values are actually part of the contract. +func TestGameFromDetected_Icarus_ProducesReadmeEquivalentValues(t *testing.T) { dir := t.TempDir() detected := steam.DetectedGame{ Slug: "icarus", diff --git a/internal/source/steam/data/steam-games.yaml b/internal/source/steam/data/steam-games.yaml index edc69fb..acc05b7 100644 --- a/internal/source/steam/data/steam-games.yaml +++ b/internal/source/steam/data/steam-games.yaml @@ -2,9 +2,9 @@ # Add or override entries in ~/.config/lmm/steam-games.yaml without rebuilding. # Keys are Steam App IDs (strings). mod_path is relative to game install (empty = game root). # nexus_id is optional (omit for games with no NexusMods presence, e.g. Icarus). -# deploy_mode and sources are optional, generated-games.yaml passthroughs -# (games.yaml's deploy_mode/sources: see docs/configuration.md); omit both to -# get today's default shape ({nexusmods: }, deploy_mode extract). +# deploy_mode and sources are optional passthroughs to the generated +# games.yaml entry's own deploy_mode/sources (see docs/configuration.md); +# omit both to get today's default shape ({nexusmods: }, deploy_mode extract). "489830": slug: skyrim-se name: Skyrim Special Edition From f45e6fb70900d838d488d7d3d8c61ae965da3b78 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 13:42:52 -0400 Subject: [PATCH 33/96] fix: cap and bound per-entry zip reads in .EXMODZ archives (security, #136) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit readZipFile now rejects any entry whose UncompressedSize64 declares more than a 64 MiB per-entry cap before reading any content, and bounds the actual read via io.LimitReader so a header that instead LIES by understating its real decompressed size can't drive an unbounded read either. Mirrors the dump-tar cap pattern from #171 review round 2 (maxTarEntrySize/fetchTree). .EXMODZ archives are user-downloaded, third-party content, unlike Icarus's own shipped paks — this closes the zip-bomb-memory-exhaustion gap the Task 11 review flagged as Minor #2. New tests: TestParseExmodz_RejectsOversizedAssetDeclaredSize (oversized header, built via zw.CreateRaw so no real 64+ MiB fixture is needed), TestParseExmodz_RejectsLyingAssetDeclaredSize (declared size under the cap but real content exceeds it). --- internal/source/icarus/exmodz.go | 27 ++++++++- internal/source/icarus/exmodz_test.go | 84 +++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/internal/source/icarus/exmodz.go b/internal/source/icarus/exmodz.go index bd47ff9..cf85338 100644 --- a/internal/source/icarus/exmodz.go +++ b/internal/source/icarus/exmodz.go @@ -91,11 +91,36 @@ func normalizeZipName(name string) string { return strings.ReplaceAll(name, `\`, "/") } +// maxZipEntrySize caps a single .EXMODZ entry's declared (and actual) +// decompressed size, mirroring #136's dump-tar cap (maxTarEntrySize): the +// largest real Icarus base data table is 7.3 MB, and .EXMODZ manifests and +// bundled UE assets are smaller still in every real sample seen. 64 MiB +// leaves generous headroom while refusing to trust an unbounded or lying +// UncompressedSize64 in a third-party, user-downloaded zip archive. +const maxZipEntrySize = 64 << 20 + func readZipFile(f *zip.File) ([]byte, error) { + if f.UncompressedSize64 > maxZipEntrySize { + return nil, fmt.Errorf("icarus: zip entry %s declares a %d-byte uncompressed size, "+ + "exceeding the %d-byte per-entry cap", f.Name, f.UncompressedSize64, uint64(maxZipEntrySize)) + } rc, err := f.Open() if err != nil { return nil, err } defer rc.Close() //nolint:errcheck - return io.ReadAll(rc) + // The cap above only guards an honest-but-large size field; a header + // that LIES by understating the real decompressed size must not be able + // to drive an unbounded read either, so the read itself is bounded one + // byte past what was declared — enough to detect an overrun without + // reading further than necessary to prove it. + data, err := io.ReadAll(io.LimitReader(rc, int64(f.UncompressedSize64)+1)) + if err != nil { + return nil, err + } + if uint64(len(data)) > f.UncompressedSize64 { + return nil, fmt.Errorf("icarus: zip entry %s decompresses past its declared %d-byte size", + f.Name, f.UncompressedSize64) + } + return data, nil } diff --git a/internal/source/icarus/exmodz_test.go b/internal/source/icarus/exmodz_test.go index e032e15..f81d980 100644 --- a/internal/source/icarus/exmodz_test.go +++ b/internal/source/icarus/exmodz_test.go @@ -3,6 +3,7 @@ package icarus import ( "archive/zip" "bytes" + "hash/crc32" "strings" "testing" ) @@ -172,3 +173,86 @@ func TestParseExmodz_NoManifest_Errors(t *testing.T) { t.Error("expected error when no .EXMOD manifest is present, got nil") } } + +// An entry (manifest or asset) declaring an uncompressed size over the +// per-entry cap must be rejected before any content is read — guards +// against a user-downloaded, third-party .EXMODZ with a corrupt or lying +// size field driving an unbounded allocation, mirroring #136's dump-tar +// cap. zw.CreateRaw writes the caller-declared size fields verbatim (unlike +// zw.Create/CreateHeader, which recompute them from what's actually +// written), so the fixture never needs 64+ real MiB of content. +func TestParseExmodz_RejectsOversizedAssetDeclaredSize(t *testing.T) { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + + w, err := zw.Create("Extracted Mods/X.EXMOD") + if err != nil { + t.Fatal(err) + } + w.Write([]byte(`{"name":"X","Rows":[]}`)) //nolint:errcheck + + content := []byte("tiny") + rawW, err := zw.CreateRaw(&zip.FileHeader{ + Name: "Bear_Mount/huge.uasset", + Method: zip.Store, + UncompressedSize64: maxZipEntrySize + 1, + CompressedSize64: uint64(len(content)), + CRC32: crc32.ChecksumIEEE(content), + }) + if err != nil { + t.Fatal(err) + } + rawW.Write(content) //nolint:errcheck + + if err := zw.Close(); err != nil { + t.Fatal(err) + } + + _, err = ParseExmodz(buf.Bytes()) + if err == nil { + t.Fatal("expected an error for an asset declaring an oversized uncompressed size, got nil") + } + if !strings.Contains(err.Error(), "Bear_Mount/huge.uasset") { + t.Errorf("error %q should name the offending entry", err) + } +} + +// An entry whose declared size is UNDER the cap (so the pre-check passes) +// but whose actual decompressed content exceeds that declared size must +// still be caught — the read itself has to be bounded, not just the +// pre-check, so a lying-but-small header can't drive an unbounded read. +func TestParseExmodz_RejectsLyingAssetDeclaredSize(t *testing.T) { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + + w, err := zw.Create("Extracted Mods/X.EXMOD") + if err != nil { + t.Fatal(err) + } + w.Write([]byte(`{"name":"X","Rows":[]}`)) //nolint:errcheck + + content := []byte("hello world") // 11 real bytes + rawW, err := zw.CreateRaw(&zip.FileHeader{ + Name: "Bear_Mount/lying.uasset", + Method: zip.Store, + UncompressedSize64: 3, // lies: declares far less than the 11 real bytes + CompressedSize64: uint64(len(content)), + CRC32: crc32.ChecksumIEEE(content), + }) + if err != nil { + t.Fatal(err) + } + rawW.Write(content) //nolint:errcheck + + if err := zw.Close(); err != nil { + t.Fatal(err) + } + + _, err = ParseExmodz(buf.Bytes()) + if err == nil { + t.Fatal("expected an error for an asset whose real content exceeds its declared uncompressed size, got nil") + } + if !strings.Contains(err.Error(), "Bear_Mount/lying.uasset") { + t.Errorf("error %q should name the offending entry", err) + } +} From 551d13f4e476cde9cd9ea13bf9a668ea8b578165 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 13:43:41 -0400 Subject: [PATCH 34/96] fix: wrap manifest-parse errors, drain Firestore error bodies (#136) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit exmodz.go: ParseExmodz now wraps a manifest-parse failure with the manifest's own zip path (icarus: %s: %w) instead of returning ParseExmod's error bare — every other error path in the function already did this (Task 11 review Minor #3); a caller couldn't previously tell from the error text alone which entry failed to parse. firestore_client.go: getJSON's non-200 error path now drains the response body to EOF before Close, instead of returning immediately with it unread. net/http only pools an HTTP/1.x connection once its body has been fully read; leaving it unread forces the transport to close the connection instead of reusing it for the source's next request (connection-reuse hygiene, behavior-neutral otherwise). New tests: TestParseExmodz_MalformedManifest_Errors, TestFirestoreClient_GetJSON_DrainsBodyBeforeCloseOnErrorPath (via a tracking RoundTripper, independent of real TCP pooling timing), TestFirestoreClient_GetDocument_Success, and TestNewFirestoreClient_NilHTTPClient_FallsBackToDefault. --- internal/source/icarus/exmodz.go | 2 +- internal/source/icarus/exmodz_test.go | 24 ++++ internal/source/icarus/firestore_client.go | 7 ++ .../source/icarus/firestore_client_test.go | 108 ++++++++++++++++++ 4 files changed, 140 insertions(+), 1 deletion(-) diff --git a/internal/source/icarus/exmodz.go b/internal/source/icarus/exmodz.go index cf85338..c666de3 100644 --- a/internal/source/icarus/exmodz.go +++ b/internal/source/icarus/exmodz.go @@ -60,7 +60,7 @@ func ParseExmodz(zipData []byte) (*ExmodzBundle, error) { } diff, err := ParseExmod(manifestData) if err != nil { - return nil, err + return nil, fmt.Errorf("icarus: %s: %w", manifestPath, err) } bundle := &ExmodzBundle{Diff: diff, Assets: make(map[string][]byte)} diff --git a/internal/source/icarus/exmodz_test.go b/internal/source/icarus/exmodz_test.go index f81d980..d8af9ab 100644 --- a/internal/source/icarus/exmodz_test.go +++ b/internal/source/icarus/exmodz_test.go @@ -174,6 +174,30 @@ func TestParseExmodz_NoManifest_Errors(t *testing.T) { } } +// A present-but-malformed manifest (invalid JSON) must fail loudly, wrapped +// with the manifest's own path — Task 11 review noted this path returned +// ParseExmod's error unwrapped, unlike every other error path in this file. +func TestParseExmodz_MalformedManifest_Errors(t *testing.T) { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, err := zw.Create("Extracted Mods/Bad.EXMOD") + if err != nil { + t.Fatal(err) + } + w.Write([]byte("{not valid json")) //nolint:errcheck + if err := zw.Close(); err != nil { + t.Fatal(err) + } + + _, err = ParseExmodz(buf.Bytes()) + if err == nil { + t.Fatal("expected an error for a malformed manifest, got nil") + } + if !strings.Contains(err.Error(), "Bad.EXMOD") { + t.Errorf("error %q should name the manifest that failed to parse", err) + } +} + // An entry (manifest or asset) declaring an uncompressed size over the // per-entry cap must be rejected before any content is read — guards // against a user-downloaded, third-party .EXMODZ with a corrupt or lying diff --git a/internal/source/icarus/firestore_client.go b/internal/source/icarus/firestore_client.go index 8fb8da7..11bb06f 100644 --- a/internal/source/icarus/firestore_client.go +++ b/internal/source/icarus/firestore_client.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" "net/url" "strings" @@ -92,6 +93,12 @@ func (c *firestoreClient) getJSON(ctx context.Context, url string, out any) erro } defer resp.Body.Close() //nolint:errcheck if resp.StatusCode != http.StatusOK { + // Drain before Close so the underlying connection stays eligible for + // reuse — net/http only pools an HTTP/1.x connection once its body + // has been read to EOF; returning immediately here left whatever the + // server sent (however small) unread, forcing the transport to + // close the connection instead of reusing it for the next request. + _, _ = io.Copy(io.Discard, resp.Body) return fmt.Errorf("HTTP %d", resp.StatusCode) } return json.NewDecoder(resp.Body).Decode(out) diff --git a/internal/source/icarus/firestore_client_test.go b/internal/source/icarus/firestore_client_test.go index 6b5bad7..4515288 100644 --- a/internal/source/icarus/firestore_client_test.go +++ b/internal/source/icarus/firestore_client_test.go @@ -3,6 +3,7 @@ package icarus import ( "context" "encoding/json" + "io" "net/http" "net/http/httptest" "testing" @@ -107,3 +108,110 @@ func TestFirestoreClient_GetDocument_NotFound(t *testing.T) { t.Fatal("expected error for 404, got nil") } } + +func TestFirestoreClient_GetDocument_Success(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + doc := map[string]any{ + "name": "projects/p/databases/(default)/documents/mods/abc", + "fields": map[string]any{"name": map[string]any{"stringValue": "Bear Mount"}}, + } + json.NewEncoder(w).Encode(doc) //nolint:errcheck + })) + defer srv.Close() + + c := newFirestoreClient("test-project", srv.Client()) + c.baseURL = srv.URL + + doc, err := c.getDocument(context.Background(), "mods", "abc") + if err != nil { + t.Fatalf("getDocument: %v", err) + } + if doc.ID != "abc" { + t.Errorf("doc.ID = %q, want abc", doc.ID) + } + if doc.Fields["name"] != "Bear Mount" { + t.Errorf("doc.Fields[name] = %v, want Bear Mount", doc.Fields["name"]) + } +} + +// newFirestoreClient(id, nil) must fall back to http.DefaultClient rather +// than leaving httpClient nil (which would panic the first time getJSON +// called c.httpClient.Do). +func TestNewFirestoreClient_NilHTTPClient_FallsBackToDefault(t *testing.T) { + c := newFirestoreClient("test-project", nil) + if c.httpClient != http.DefaultClient { + t.Errorf("httpClient = %v, want http.DefaultClient", c.httpClient) + } +} + +// trackingBody wraps a response body to record whether it was read all the +// way to io.EOF (drained) and whether Close was called, independent of real +// TCP connection pooling — the property getJSON's error path needs to +// guarantee is "drain, then close," not any particular transport behavior. +type trackingBody struct { + rc io.ReadCloser + readToEOF bool + closed bool +} + +func (b *trackingBody) Read(p []byte) (int, error) { + n, err := b.rc.Read(p) + if err == io.EOF { + b.readToEOF = true + } + return n, err +} + +func (b *trackingBody) Close() error { + b.closed = true + return b.rc.Close() +} + +// drainTrackingTransport substitutes every response's body with a +// trackingBody, recording the last one seen so a test can inspect it after +// the request completes. +type drainTrackingTransport struct { + base http.RoundTripper + tracked *trackingBody +} + +func (t *drainTrackingTransport) RoundTrip(req *http.Request) (*http.Response, error) { + resp, err := t.base.RoundTrip(req) + if err != nil { + return resp, err + } + t.tracked = &trackingBody{rc: resp.Body} + resp.Body = t.tracked + return resp, nil +} + +// getJSON's non-200 error path must drain the response body to EOF before +// closing it — an early return without draining leaves bytes unread on the +// wire, which forces net/http's transport to close the underlying +// connection instead of pooling it for reuse (connection-reuse hygiene). +func TestFirestoreClient_GetJSON_DrainsBodyBeforeCloseOnErrorPath(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(`{"error":"nope, and some padding so there is real body to drain"}`)) //nolint:errcheck + })) + defer srv.Close() + + transport := &drainTrackingTransport{base: http.DefaultTransport} + client := &http.Client{Transport: transport} + c := newFirestoreClient("test-project", client) + c.baseURL = srv.URL + + err := c.getJSON(context.Background(), srv.URL, &struct{}{}) + if err == nil { + t.Fatal("expected an error for a non-200 response, got nil") + } + if transport.tracked == nil { + t.Fatal("transport never saw a request") + } + if !transport.tracked.readToEOF { + t.Error("response body was not drained to EOF before Close on the error path") + } + if !transport.tracked.closed { + t.Error("response body was not closed") + } +} From 4c2ede94f5f65a2ebe0c3fc09e7550d86d41efde Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 13:44:05 -0400 Subject: [PATCH 35/96] test: broaden defensive and API coverage sweep (#136) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit internal/unrealpak (zlib_test.go): 3 regression tests for already-correct defensive paths flagged as deferred-but-OK-to-merge in the #175 final review's ledger triage — a block decompressing past its declared UncompressedSize, an entry declaring a size over maxUncompressedEntrySize, and a block span (start/end) falling outside its entry's own payload region. zlibFixture gained an optional declaredUncompressed override so a fixture can lie about its size without needing real gigabytes of content. internal/source/icarus: - firestore_value_test.go: table-drives decodeValue across every value kind it handles (stringValue, booleanValue, integerValue, doubleValue, mapValue, arrayValue, nullValue) plus both fallback-to-nil paths — only stringValue/mapValue/nullValue had coverage before, via decodeFields. - icarus_test.go: mock-HTTP tests for GetMod (happy path), GetDownloadURL (table-driven: pak/exmodz resolve, unrecognized fileID errors), and CheckUpdates (a mix of an outdated and up-to-date installed mod against a per-ID catalog) — all three had zero prior coverage. Does not touch GetMod's ignored gameID, Category/Tags filtering, or primary-file marking (tracked/accepted elsewhere). --- .../source/icarus/firestore_value_test.go | 47 +++++++ internal/source/icarus/icarus_test.go | 121 ++++++++++++++++++ internal/unrealpak/zlib_test.go | 103 +++++++++++++++ 3 files changed, 271 insertions(+) diff --git a/internal/source/icarus/firestore_value_test.go b/internal/source/icarus/firestore_value_test.go index 47882e3..a25a155 100644 --- a/internal/source/icarus/firestore_value_test.go +++ b/internal/source/icarus/firestore_value_test.go @@ -32,3 +32,50 @@ func TestDecodeFields(t *testing.T) { t.Errorf("decodeFields() = %#v, want %#v", got, want) } } + +// TestDecodeValue table-drives every value kind decodeValue recognizes, +// plus its two fallback-to-nil paths (an unrecognized kind, and input that +// isn't a wrapped {"kind": ...} object at all) — decodeFields' own test +// above only exercises stringValue/mapValue/nullValue. +func TestDecodeValue(t *testing.T) { + tests := []struct { + name string + in any + want any + }{ + {"stringValue", map[string]any{"stringValue": "hello"}, "hello"}, + {"booleanValue", map[string]any{"booleanValue": true}, true}, + // Firestore's REST API encodes integerValue as a decimal STRING + // (avoiding int64-precision loss in JSON numbers), not a JSON number. + {"integerValue", map[string]any{"integerValue": "42"}, "42"}, + {"doubleValue", map[string]any{"doubleValue": 3.5}, 3.5}, + { + "mapValue decodes its nested fields recursively", + map[string]any{"mapValue": map[string]any{"fields": map[string]any{ + "inner": map[string]any{"stringValue": "nested"}, + }}}, + map[string]any{"inner": "nested"}, + }, + { + "arrayValue decodes each element recursively", + map[string]any{"arrayValue": map[string]any{"values": []any{ + map[string]any{"stringValue": "a"}, + map[string]any{"integerValue": "1"}, + map[string]any{"booleanValue": false}, + }}}, + []any{"a", "1", false}, + }, + {"arrayValue with no values key decodes to an empty slice", map[string]any{"arrayValue": map[string]any{}}, []any{}}, + {"nullValue", map[string]any{"nullValue": nil}, nil}, + {"unrecognized kind decodes to nil, not a panic", map[string]any{"geoPointValue": map[string]any{"latitude": 1.0}}, nil}, + {"non-map input decodes to nil", "not a wrapped value", nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := decodeValue(tt.in) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("decodeValue(%#v) = %#v, want %#v", tt.in, got, tt.want) + } + }) + } +} diff --git a/internal/source/icarus/icarus_test.go b/internal/source/icarus/icarus_test.go index 399f45e..061163d 100644 --- a/internal/source/icarus/icarus_test.go +++ b/internal/source/icarus/icarus_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "path" "testing" "github.com/DonovanMods/linux-mod-manager/internal/domain" @@ -107,6 +108,126 @@ func TestIcarus_GetModFiles_ReturnsExmodzAndPak(t *testing.T) { } } +// TestIcarus_GetMod pins the happy path: a single Firestore document maps +// through mapDoc into the domain.Mod fields Search/GetModFiles callers +// expect. GetMod's queryGameID parameter is deliberately not exercised here +// (tracked/accepted elsewhere, per this sweep's scope). +func TestIcarus_GetMod(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]any{ //nolint:errcheck + "name": "projects/p/databases/(default)/documents/mods/abc", + "fields": map[string]any{ + "name": map[string]any{"stringValue": "Bear Mount"}, + "author": map[string]any{"stringValue": "Jimk72"}, + "version": map[string]any{"stringValue": "3.3"}, + }, + }) + })) + defer srv.Close() + + src := New(srv.Client(), "test-project") + src.firestore.baseURL = srv.URL + + mod, err := src.GetMod(context.Background(), "icarus", "abc") + if err != nil { + t.Fatalf("GetMod: %v", err) + } + if mod.ID != "abc" || mod.Name != "Bear Mount" || mod.Author != "Jimk72" || mod.Version != "3.3" { + t.Errorf("GetMod = %+v, want ID=abc Name=Bear Mount Author=Jimk72 Version=3.3", mod) + } + if mod.GameID != "icarus" { + t.Errorf("GameID = %q, want icarus", mod.GameID) + } +} + +// TestIcarus_GetDownloadURL table-drives both the success path (fileID +// present, either "pak" or "exmodz") and the not-found error path. +func TestIcarus_GetDownloadURL(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]any{ //nolint:errcheck + "name": "projects/p/databases/(default)/documents/mods/abc", + "fields": map[string]any{ + "files": map[string]any{"mapValue": map[string]any{"fields": map[string]any{ + "pak": map[string]any{"stringValue": "https://x/bear.pak"}, + "exmodz": map[string]any{"stringValue": "https://x/bear.exmodz"}, + }}}, + }, + }) + })) + defer srv.Close() + + src := New(srv.Client(), "test-project") + src.firestore.baseURL = srv.URL + mod := &domain.Mod{ID: "abc", GameID: "icarus"} + + tests := []struct { + name string + fileID string + want string + wantErr bool + }{ + {name: "pak file ID resolves", fileID: "pak", want: "https://x/bear.pak"}, + {name: "exmodz file ID resolves", fileID: "exmodz", want: "https://x/bear.exmodz"}, + {name: "unrecognized file ID errors", fileID: "nope", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := src.GetDownloadURL(context.Background(), mod, tt.fileID) + if tt.wantErr { + if err == nil { + t.Fatalf("GetDownloadURL(%q) = %q, nil; want an error", tt.fileID, got) + } + return + } + if err != nil { + t.Fatalf("GetDownloadURL(%q): %v", tt.fileID, err) + } + if got != tt.want { + t.Errorf("GetDownloadURL(%q) = %q, want %q", tt.fileID, got, tt.want) + } + }) + } +} + +// TestIcarus_CheckUpdates drives two installed mods against a per-ID +// catalog: one whose stored version is behind the catalog's (an update is +// expected) and one that already matches (no update). +func TestIcarus_CheckUpdates(t *testing.T) { + catalog := map[string]map[string]any{ + "abc": {"name": map[string]any{"stringValue": "Bear Mount"}, "version": map[string]any{"stringValue": "3.3"}}, + "def": {"name": map[string]any{"stringValue": "Wolf Pack"}, "version": map[string]any{"stringValue": "1.0"}}, + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + id := path.Base(r.URL.Path) + fields, ok := catalog[id] + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + json.NewEncoder(w).Encode(map[string]any{ //nolint:errcheck + "name": "projects/p/databases/(default)/documents/mods/" + id, + "fields": fields, + }) + })) + defer srv.Close() + + src := New(srv.Client(), "test-project") + src.firestore.baseURL = srv.URL + + installed := []domain.InstalledMod{ + {Mod: domain.Mod{ID: "abc", Version: "3.2"}}, // catalog has 3.3: newer, update expected + {Mod: domain.Mod{ID: "def", Version: "1.0"}}, // catalog has 1.0: same, no update + } + + updates, err := src.CheckUpdates(context.Background(), installed) + if err != nil { + t.Fatalf("CheckUpdates: %v", err) + } + if len(updates) != 1 || updates[0].InstalledMod.ID != "abc" || updates[0].NewVersion != "3.3" { + t.Fatalf("updates = %+v, want exactly one update for abc -> 3.3", updates) + } +} + // The fallback must always be a dotted name (never a bare "exmodz"/"pak"), // or isExmodzFile's ".exmodz" suffix check and compiledFileName's // filepath.Ext-based rename in Service would both silently misroute the diff --git a/internal/unrealpak/zlib_test.go b/internal/unrealpak/zlib_test.go index 86a69a2..78e3610 100644 --- a/internal/unrealpak/zlib_test.go +++ b/internal/unrealpak/zlib_test.go @@ -17,6 +17,14 @@ type zlibFixture struct { path string blocks [][]byte // each block's PLAINTEXT; each is deflated independently method int32 // 1-based index into the methods table below + // declaredUncompressed, when non-zero, overrides the UncompressedSize + // this fixture declares in both its on-disk header and its encoded + // index record, independent of the real sum of block plaintext lengths + // — for fixtures that need to lie about their size (decompression-bomb + // and size-cap regression tests) without needing real gigabytes of + // content. Zero means "use the real sum," which every genuine fixture + // in this file has anyway (none declares a real zero-length entry). + declaredUncompressed int64 } // writeMethodPak hand-builds a version-11 pak whose footer declares methods and @@ -54,6 +62,9 @@ func writeMethodPak(t *testing.T, methods []string, fixtures []zlibFixture) stri spans = append(spans, span{start, hdrSize + int64(payload.Len())}) uncompressed += int64(len(plain)) } + if fx.declaredUncompressed != 0 { + uncompressed = fx.declaredUncompressed + } size := int64(payload.Len()) sum := sha1.Sum(payload.Bytes()) //nolint:gosec @@ -265,3 +276,95 @@ func TestReadFile_ZlibCorruptPayload_FailsHashGate(t *testing.T) { t.Fatal("expected an error for a corrupted compressed payload, got nil") } } + +// A block that decompresses to more bytes than the entry's own declared +// UncompressedSize (a lying header, or an over-inflating compression bomb) +// must be caught mid-decompress, not silently truncated or allowed to +// over-allocate — #175 final review ledger item 1. The real block content +// is genuinely 1000 bytes; the fixture just declares (and the index/header +// therefore both agree on) a 5-byte size, which is what readZlib actually +// checks against as it reads. +func TestReadFile_ZlibBlockExceedsDeclaredSize_Errors(t *testing.T) { + huge := bytes.Repeat([]byte("A"), 1000) + p := writeMethodPak(t, []string{"Zlib"}, []zlibFixture{ + {path: "bomb/D.json", blocks: [][]byte{huge}, method: 1, declaredUncompressed: 5}, + }) + + r, err := Open(p) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() //nolint:errcheck + + _, err = r.ReadFile("bomb/D.json") + if err == nil { + t.Fatal("expected an error for a block decompressing past the declared uncompressed size, got nil") + } + if !strings.Contains(err.Error(), "exceeds the declared uncompressed size") { + t.Errorf("error %q should name the size mismatch", err) + } +} + +// An entry declaring an UncompressedSize over maxUncompressedEntrySize must +// be refused before any decompression is attempted — #175 final review +// ledger item 2. The real block content is tiny; only the declared size +// needs to be oversized; the cap check runs before the block loop ever +// touches the payload, so no gigabytes of real content are needed. +func TestReadFile_ZlibUncompressedSizeExceedsCap_IsUnsupported(t *testing.T) { + body := []byte("tiny") + p := writeMethodPak(t, []string{"Zlib"}, []zlibFixture{ + {path: "big/Huge.json", blocks: [][]byte{body}, method: 1, declaredUncompressed: maxUncompressedEntrySize + 1}, + }) + + r, err := Open(p) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() //nolint:errcheck + + _, err = r.ReadFile("big/Huge.json") + if !errors.Is(err, ErrUnsupportedFormat) { + t.Fatalf("err = %v, want ErrUnsupportedFormat", err) + } + if !strings.Contains(err.Error(), "exceeds the") { + t.Errorf("error %q should name the cap", err) + } +} + +// A block whose (start, end) span falls outside the entry's own payload +// region must be refused, never sliced out of range — #175 final review +// ledger item 3. Corrupts the single block's "end" field (at header byte +// offset 60, per readZlib's hdr[60+i*16:68+i*16] for i=0) to 0, which is +// less than "start" (== hdrSize == compressedHeaderSize(1), unaffected by +// this edit), tripping the end < start bounds check — independent of the +// SHA1 hash gate, since only the header's block-span table is touched, not +// the actual compressed payload bytes the hash covers. +func TestReadFile_ZlibBlockSpanOutsidePayload_Errors(t *testing.T) { + p := writeMethodPak(t, []string{"Zlib"}, []zlibFixture{ + {path: "c/D.json", blocks: [][]byte{[]byte("hello world")}, method: 1}, + }) + raw, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + for i := 60; i < 68; i++ { + raw[i] = 0 + } + if err := os.WriteFile(p, raw, 0o644); err != nil { + t.Fatal(err) + } + + r, err := Open(p) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() //nolint:errcheck + + _, err = r.ReadFile("c/D.json") + if err == nil { + t.Fatal("expected an error for a block span outside the entry's payload, got nil") + } + if !strings.Contains(err.Error(), "outside the entry's payload") { + t.Errorf("error %q should name the bounds violation", err) + } +} From 3d30637f759a50e8c57904fa6f27b1c76a571208 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 13:50:09 -0400 Subject: [PATCH 36/96] fix: clamp search pagination overflow, correct stale base-pak comment (review) (#136) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Icarus.Search: page*pageSize (computing "start") and start+pageSize (computing "end") can each overflow int for a huge user-supplied Page or PageSize, wrapping to a negative value the existing "> len(mods)" clamps never catch (they only guard the too-large direction) — panicking the final mods[start:end] slice instead of just returning an empty page past the end of the result set. Both multiplication and addition are now guarded before they run (division-based overflow checks, safe since pageSize > 0 is already guaranteed by the clamp above them) rather than trusted to land in range. internal/core/service.go: resolveBasePak's doc comment still described the rev3 hosted-dump design ("this pak is no longer the source of base table content... that comes from the hosted dump") — stale since #175 deleted the dump subsystem entirely. Reworded to the current truth: Compile reads base tables straight out of this pak, offline and always week-correct by construction. Also dropped a version.json cross-reference left over from the now-deleted detectBuild. New tests: TestIcarus_Search_HugePage_ClampsInsteadOfPanicking (the page*pageSize overflow) and TestIcarus_Search_HugePageSize_ClampsInsteadOfPanicking (the start+pageSize overflow, which needs Page:1 specifically — Page:0 can't exercise it, since 0+anything never overflows). Both panicked against the unmodified code (RED), pass after the fix (GREEN). --- internal/core/service.go | 8 ++-- internal/source/icarus/icarus.go | 27 +++++++++++--- internal/source/icarus/icarus_test.go | 53 +++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 10 deletions(-) diff --git a/internal/core/service.go b/internal/core/service.go index 67f6d67..2bdd2e4 100644 --- a/internal/core/service.go +++ b/internal/core/service.go @@ -940,10 +940,10 @@ func isExmodzFile(fileName string) bool { // JSON data tables live in Content/Data/data.pak, NOT in the Content/Paks // pakchunks, which carry only cooked .uasset/.uexp assets and no JSON at all. // -// Since rev3 this pak is no longer the source of base table *content* (that -// comes from the hosted dump — Task 12a); it is still required, because it is -// the only authority on which tables the installed game has and on which game -// week is installed. Its parent directory also locates Icarus/Config/version.json. +// This pak is also the direct source of base table *content* (#175): Compile +// reads each patched table straight out of it via internal/unrealpak, so a +// compile is always week-correct by construction (there's no separate dump +// to go stale relative to the install) and works entirely offline. func resolveBasePak(game *domain.Game) (string, error) { candidate := filepath.Join(game.InstallPath, "Icarus", "Content", "Data", "data.pak") if _, err := os.Stat(candidate); err != nil { diff --git a/internal/source/icarus/icarus.go b/internal/source/icarus/icarus.go index 029505d..acdfb93 100644 --- a/internal/source/icarus/icarus.go +++ b/internal/source/icarus/icarus.go @@ -3,6 +3,7 @@ package icarus import ( "context" "fmt" + "math" "net/http" "net/url" "path" @@ -104,13 +105,27 @@ func (s *Icarus) Search(ctx context.Context, query source.SearchQuery) (source.S if page < 0 { page = 0 } - start := page * pageSize - if start > len(mods) { - start = len(mods) + // Both page*pageSize (below) and start+pageSize (for end, further down) + // can overflow int for a huge user-supplied Page or PageSize, wrapping + // negative instead of landing somewhere past len(mods) — the pre-fix + // "start > len(mods)"/"end > len(mods)" clamps never catch a NEGATIVE + // value, so a wrapped result panicked the slice below instead of just + // clamping to an empty page. Both operations are guarded before they + // run rather than trusted to land in range; pageSize > 0 is guaranteed + // by the clamp above, so both divisions here are always safe. + start := len(mods) + if page <= math.MaxInt/pageSize { + start = page * pageSize + if start > len(mods) { + start = len(mods) + } } - end := start + pageSize - if end > len(mods) { - end = len(mods) + end := len(mods) + if pageSize <= math.MaxInt-start { + end = start + pageSize + if end > len(mods) { + end = len(mods) + } } return source.SearchResult{Mods: mods[start:end], TotalCount: len(mods), Page: page, PageSize: pageSize}, nil diff --git a/internal/source/icarus/icarus_test.go b/internal/source/icarus/icarus_test.go index 061163d..6b88102 100644 --- a/internal/source/icarus/icarus_test.go +++ b/internal/source/icarus/icarus_test.go @@ -3,6 +3,7 @@ package icarus import ( "context" "encoding/json" + "math" "net/http" "net/http/httptest" "path" @@ -79,6 +80,58 @@ func TestIcarus_Search_FiltersClientSide(t *testing.T) { } } +// A huge user-supplied Page overflows the page*pageSize multiplication +// (int wraps to a negative value), which previously produced a negative +// slice start and panicked instead of just clamping to an empty page past +// the end of the result set (#136 PR #181 review). +func TestIcarus_Search_HugePage_ClampsInsteadOfPanicking(t *testing.T) { + srv := httptest.NewServer(modsListHandler([]map[string]any{ + {"id": "abc", "fields": map[string]any{"name": map[string]any{"stringValue": "Bear Mount"}}}, + })) + defer srv.Close() + + src := New(srv.Client(), "test-project") + src.firestore.baseURL = srv.URL + + result, err := src.Search(context.Background(), source.SearchQuery{Page: math.MaxInt, PageSize: 20}) + if err != nil { + t.Fatalf("Search: %v", err) + } + if len(result.Mods) != 0 { + t.Errorf("Mods = %+v, want empty (page far beyond the 1-mod result set)", result.Mods) + } + if result.TotalCount != 1 { + t.Errorf("TotalCount = %d, want 1", result.TotalCount) + } +} + +// The same overflow class hits start+pageSize (computing "end") once start +// is non-zero: page*pageSize (1*MaxInt) doesn't itself overflow, and gets +// clamped down to the small, valid len(mods) — but THAT small start plus a +// huge PageSize wraps the addition negative, and the existing +// "end > len(mods)" clamp can't catch an end that's gone negative (Page: 0 +// can't exercise this: 0+anything never overflows). +func TestIcarus_Search_HugePageSize_ClampsInsteadOfPanicking(t *testing.T) { + srv := httptest.NewServer(modsListHandler([]map[string]any{ + {"id": "abc", "fields": map[string]any{"name": map[string]any{"stringValue": "Bear Mount"}}}, + })) + defer srv.Close() + + src := New(srv.Client(), "test-project") + src.firestore.baseURL = srv.URL + + result, err := src.Search(context.Background(), source.SearchQuery{Page: 1, PageSize: math.MaxInt}) + if err != nil { + t.Fatalf("Search: %v", err) + } + if len(result.Mods) != 0 { + t.Errorf("Mods = %+v, want empty (page 1 is past the 1-mod result set)", result.Mods) + } + if result.TotalCount != 1 { + t.Errorf("TotalCount = %d, want 1", result.TotalCount) + } +} + func TestIcarus_GetModFiles_ReturnsExmodzAndPak(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(map[string]any{ //nolint:errcheck From 1b7afb3ec5309d9d795df79ef037d917e56a665a Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 14:41:56 -0400 Subject: [PATCH 37/96] fix: record deployed state on mod disable/enable (#183) DisableMod undeployed files and cleared enabled, but never cleared deployed, so `lmm list -v` kept showing DEPLOYED yes after disable. EnableMod had the symmetric gap on re-deploy. Both flows now call SetModDeployed after the (un)deploy step, using the same non-fatal Note convention DeployProfile/PurgeProfile already use for this setter: a failed flag write doesn't block the primary enable/disable outcome. DisableMod clears deployed unconditionally, even when Uninstall only partially succeeds, so the flag always tracks disable-intent rather than lagging a best-effort file cleanup. --- CHANGELOG.md | 4 ++ cmd/lmm/mod.go | 9 ++--- internal/core/flows.go | 66 +++++++++++++++++++++++++-------- internal/core/flows_test.go | 74 ++++++++++++++++++++++++++++++++++++- 4 files changed, 132 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 703923d..878cf73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- `lmm mod disable` undeployed a mod's files and cleared `enabled`, but never cleared `deployed` — `lmm list -v` kept showing DEPLOYED yes after disable. The disable flow now clears `deployed` unconditionally after the undeploy attempt, even when the undeploy itself only partially succeeds (already a non-fatal, Note-reported condition), so the flag always reflects disable-intent rather than lagging behind a best-effort file cleanup. The symmetric enable path had the same gap — enabling a disabled mod re-deployed its files without ever setting `deployed` back to true — and is fixed the same way. Both `SetModDeployed` calls follow the same non-fatal Note convention already used by `DeployProfile`/`PurgeProfile` for this same setter: a failure to record the flag doesn't block the primary enable/disable outcome (#183) + ## [1.27.1] - 2026-07-30 ### Fixed diff --git a/cmd/lmm/mod.go b/cmd/lmm/mod.go index 8e50a4e..7e47fc6 100644 --- a/cmd/lmm/mod.go +++ b/cmd/lmm/mod.go @@ -405,11 +405,10 @@ func doModEnable(ctx context.Context, service *core.Service, game *domain.Game, // accumulated before the fatal error alongside it (mirrors // UninstallMod's own convention - see uninstall.go's // printUninstallDiagnostics); print them now, or they'd otherwise be - // lost even though they already happened. result is nil on every - // EnableMod error path today, but is guarded here anyway, for parity - // with doModDisable below and because EnableResult.Notes is kept - // specifically so a future EnableMod diagnostic wouldn't need - // another signature change (see its doc comment in flows.go). + // lost even though they already happened (e.g. a SetModDeployed + // failure Note recorded, then a later SetModEnabled failure, #183). + // result is nil only when EnableMod failed before it could allocate + // the result struct, exactly like doModDisable below. if result != nil { printModNotes(result.Notes) } diff --git a/internal/core/flows.go b/internal/core/flows.go index fbab2c8..c73d6a3 100644 --- a/internal/core/flows.go +++ b/internal/core/flows.go @@ -22,29 +22,42 @@ import ( // error) return. Notes carries operational diagnostics using the same // display-contract convention as UninstallResult/DeployResult (Task 2's // convention, extended here in Task 6 item a for result-struct -// convergence): always empty today — EnableMod has no diagnostic-producing -// step — kept for parity with DisableResult and so a future EnableMod -// diagnostic wouldn't need another signature change. +// convergence, and by #183's SetModDeployed note below): a caller wanting +// byte-identical pre-5a output should print each entry to stdout ONLY +// under --verbose, verbatim, e.g. `fmt.Printf(" %s\n", n)`. type EnableResult struct { Changed bool Notes []string } // DisableResult reports the outcome of DisableMod. Changed mirrors -// EnableResult.Changed. Notes carries the sole diagnostic DisableMod can -// produce — a non-fatal undeploy failure (see DisableMod's doc comment) — -// using the same historical-prefix-baked-into-the-text convention -// UninstallResult's doc comment documents: a caller wanting byte-identical -// pre-5a output should print each entry to stdout ONLY under --verbose, -// verbatim, e.g. `fmt.Printf(" %s\n", n)`. +// EnableResult.Changed. Notes carries the diagnostics DisableMod can +// produce — a non-fatal undeploy failure (see DisableMod's doc comment) and +// (#183) a non-fatal SetModDeployed failure — using the same +// historical-prefix-baked-into-the-text convention UninstallResult's doc +// comment documents: a caller wanting byte-identical pre-5a output should +// print each entry to stdout ONLY under --verbose, verbatim, e.g. +// `fmt.Printf(" %s\n", n)`. type DisableResult struct { Changed bool Notes []string } // EnableMod deploys an installed-but-disabled mod's files from the cache to -// the game directory and marks it enabled in the database. Returns a result -// with Changed false — not an error — if the mod was already enabled. +// the game directory and marks it enabled (and deployed, #183) in the +// database. Returns a result with Changed false — not an error — if the +// mod was already enabled. +// +// A SetModDeployed failure is non-fatal (recorded in Notes) — mirroring +// both DisableMod's own treatment of the identical call and +// DeployProfile's/PurgeProfile's existing SetModDeployed call sites: the +// files are already live on disk at this point, and refusing to record the +// user's intent to enable the mod over a secondary bookkeeping-write +// failure would leave it stuck exactly like the undeploy-failure case +// DisableMod already accepts. SetModEnabled's own failure, in contrast, +// stays fatal (pre-existing behavior, unchanged) — it is the write that +// makes "the mod is enabled" true at all, unlike the deployed flag, which +// is a cache of already-true, already-observable state. func (s *Service) EnableMod(ctx context.Context, game *domain.Game, profileName, sourceID, modID string) (*EnableResult, error) { mod, err := s.GetInstalledMod(sourceID, modID, game.ID, profileName) if err != nil { @@ -64,17 +77,24 @@ func (s *Service) EnableMod(ctx context.Context, game *domain.Game, profileName, return nil, fmt.Errorf("failed to deploy mod: %w", err) } + result := &EnableResult{} + if err := s.SetModDeployed(sourceID, modID, game.ID, profileName, true); err != nil { + result.Notes = append(result.Notes, fmt.Sprintf("Warning: could not mark as deployed: %v", err)) + } + if err := s.SetModEnabled(sourceID, modID, game.ID, profileName, true); err != nil { - return nil, fmt.Errorf("failed to update mod status: %w", err) + return result, fmt.Errorf("failed to update mod status: %w", err) } - return &EnableResult{Changed: true}, nil + result.Changed = true + return result, nil } // DisableMod undeploys the mod's files from the game directory — the cache // entry is kept so the mod can be re-enabled later without downloading again -// — and marks it disabled in the database. Returns a result with Changed -// false — not an error — if the mod was already disabled. +// — and marks it disabled (and not-deployed, #183) in the database. Returns +// a result with Changed false — not an error — if the mod was already +// disabled. // // Undeploy failures are treated as non-fatal: the game files may already // have been removed manually, and refusing to record the user's intent to @@ -83,6 +103,18 @@ func (s *Service) EnableMod(ctx context.Context, game *domain.Game, profileName, // — DisableResult.Notes (Task 6 item a) restores that diagnostic for // callers that want it, rather than discarding it as the (bool, error) // signature this replaces was forced to. +// +// A SetModDeployed failure gets the identical non-fatal treatment (#183), +// for the identical reason and matching DeployProfile's/PurgeProfile's own +// SetModDeployed call sites: it is attempted unconditionally, even when the +// undeploy above already failed, because the deployed flag should reflect +// "disable was requested" regardless of whether the file-level undeploy +// itself succeeded — an undeploy failure already means the flag may not +// match reality either way, and the alternative (skipping the write +// because undeploy failed) would leave the mod stuck reporting DEPLOYED +// forever, which is #183 itself. SetModEnabled's own failure stays fatal, +// unchanged: it is the write that makes "the mod is disabled" true at all, +// not a cache of already-true state. func (s *Service) DisableMod(ctx context.Context, game *domain.Game, profileName, sourceID, modID string) (*DisableResult, error) { mod, err := s.GetInstalledMod(sourceID, modID, game.ID, profileName) if err != nil { @@ -101,6 +133,10 @@ func (s *Service) DisableMod(ctx context.Context, game *domain.Game, profileName result.Notes = append(result.Notes, fmt.Sprintf("Warning: failed to undeploy some files: %v", err)) } + if err := s.SetModDeployed(sourceID, modID, game.ID, profileName, false); err != nil { + result.Notes = append(result.Notes, fmt.Sprintf("Warning: could not mark as not deployed: %v", err)) + } + if err := s.SetModEnabled(sourceID, modID, game.ID, profileName, false); err != nil { return result, fmt.Errorf("failed to update mod status: %w", err) } diff --git a/internal/core/flows_test.go b/internal/core/flows_test.go index ac49559..fd09c5f 100644 --- a/internal/core/flows_test.go +++ b/internal/core/flows_test.go @@ -128,6 +128,7 @@ func TestService_EnableMod_DeploysDisabledMod(t *testing.T) { mod, err := svc.GetInstalledMod("src", "1", "g1", "default") require.NoError(t, err) assert.True(t, mod.Enabled) + assert.True(t, mod.Deployed, "#183: enabling a mod must also record it as deployed") } func TestService_EnableMod_AlreadyEnabledIsNoop(t *testing.T) { @@ -214,9 +215,12 @@ func TestService_DisableMod_UndeploysEnabledMod(t *testing.T) { }) // Deploy the files first so there's something to undeploy (mirrors an - // install that happened earlier). + // install that happened earlier), and record the DB's deployed flag to + // match — seedInstalledMod doesn't set it, so this mirrors the real + // precondition DisableMod actually sees for a genuinely-deployed mod. installer := svc.GetInstaller(game) require.NoError(t, installer.Install(context.Background(), game, &domain.Mod{ID: "1", SourceID: "src", Version: "1.0", GameID: "g1"}, "default")) + require.NoError(t, svc.SetModDeployed("src", "1", "g1", "default", true)) result, err := svc.DisableMod(context.Background(), game, "default", "src", "1") require.NoError(t, err) @@ -232,6 +236,7 @@ func TestService_DisableMod_UndeploysEnabledMod(t *testing.T) { mod, err := svc.GetInstalledMod("src", "1", "g1", "default") require.NoError(t, err) assert.False(t, mod.Enabled) + assert.False(t, mod.Deployed, "#183: disabling a mod must also clear the deployed flag") } func TestService_DisableMod_AlreadyDisabledIsNoop(t *testing.T) { @@ -297,6 +302,73 @@ func TestService_DisableMod_UndeployFailureIsNonFatal(t *testing.T) { mod, err := svc.GetInstalledMod("src", "1", "g1", "default") require.NoError(t, err) assert.False(t, mod.Enabled, "DB should still flip to disabled even when undeploy is best-effort") + assert.False(t, mod.Deployed, "#183: the deployed flag must still clear to record intent, even when the undeploy itself was best-effort") +} + +// TestService_DisableMod_SetModDeployedFailure_NonFatalNote pins the #183 +// fix's own failure-handling decision: a SetModDeployed(false) failure is +// non-fatal, recorded as a Note, exactly like DisableMod's existing +// Uninstall-failure handling and like PurgeProfile's own SetModDeployed +// call (see TestService_PurgeProfile_SetModDeployedFailure_NonFatalNote) — +// not escalated to a hard error the way SetModEnabled's own failure still +// is. installBlockingTrigger blocks only the "deployed" column, so +// SetModEnabled (a different column) still succeeds. +func TestService_DisableMod_SetModDeployedFailure_NonFatalNote(t *testing.T) { + dataDir := t.TempDir() + svc, err := core.NewService(core.ServiceConfig{ + ConfigDir: t.TempDir(), DataDir: dataDir, CacheDir: t.TempDir(), + }) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + gameDir := t.TempDir() + game := &domain.Game{ID: "g1", Name: "Game", ModPath: gameDir, LinkMethod: domain.LinkSymlink} + + seedInstalledMod(t, svc, game, "src", "1", "1.0", true, map[string][]byte{"plugin.esp": []byte("data")}) + installSeededMod(t, svc, game, "1") + installBlockingTrigger(t, filepath.Join(dataDir, "lmm.db")) + + result, err := svc.DisableMod(context.Background(), game, "default", "src", "1") + require.NoError(t, err, "a SetModDeployed failure must not fail DisableMod") + require.NotNil(t, result) + assert.True(t, result.Changed) + require.NotEmpty(t, result.Notes) + assert.Contains(t, strings.Join(result.Notes, "\n"), "could not mark as not deployed", + "Notes = %v, want one mentioning the SetModDeployed failure", result.Notes) + + mod, err := svc.GetInstalledMod("src", "1", "g1", "default") + require.NoError(t, err) + assert.False(t, mod.Enabled, "SetModEnabled touches a different column and must still succeed") +} + +// TestService_EnableMod_SetModDeployedFailure_NonFatalNote is +// TestService_DisableMod_SetModDeployedFailure_NonFatalNote's mirror for +// the enable path. +func TestService_EnableMod_SetModDeployedFailure_NonFatalNote(t *testing.T) { + dataDir := t.TempDir() + svc, err := core.NewService(core.ServiceConfig{ + ConfigDir: t.TempDir(), DataDir: dataDir, CacheDir: t.TempDir(), + }) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + gameDir := t.TempDir() + game := &domain.Game{ID: "g1", Name: "Game", ModPath: gameDir, LinkMethod: domain.LinkSymlink} + + seedInstalledMod(t, svc, game, "src", "1", "1.0", false, map[string][]byte{"plugin.esp": []byte("data")}) + installBlockingTrigger(t, filepath.Join(dataDir, "lmm.db")) + + result, err := svc.EnableMod(context.Background(), game, "default", "src", "1") + require.NoError(t, err, "a SetModDeployed failure must not fail EnableMod") + require.NotNil(t, result) + assert.True(t, result.Changed) + require.NotEmpty(t, result.Notes) + assert.Contains(t, strings.Join(result.Notes, "\n"), "could not mark as deployed", + "Notes = %v, want one mentioning the SetModDeployed failure", result.Notes) + + mod, err := svc.GetInstalledMod("src", "1", "g1", "default") + require.NoError(t, err) + assert.True(t, mod.Enabled, "SetModEnabled touches a different column and must still succeed") } // --- UninstallMod --- From e0cdf58a8fa48313e20a56535fe7a703f761995f Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 14:47:56 -0400 Subject: [PATCH 38/96] fix: seed Deployed=true in the undeploy-failure test (#183 review) TestService_DisableMod_UndeployFailureIsNonFatal asserted mod.Deployed was false after disable, but its setup never set Deployed=true in the first place, so the assertion couldn't prove SetModDeployed(false) ran on the partial-Uninstall-failure path. Seed the flag before disabling so the post-condition is a genuine transition check. RED-verified by temporarily removing the SetModDeployed(false) call in DisableMod: the test now fails as expected, then passes again once restored. --- internal/core/flows_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/core/flows_test.go b/internal/core/flows_test.go index fd09c5f..048b48f 100644 --- a/internal/core/flows_test.go +++ b/internal/core/flows_test.go @@ -284,6 +284,8 @@ func TestService_DisableMod_UndeployFailureIsNonFatal(t *testing.T) { installer := svc.GetInstaller(game) require.NoError(t, installer.Install(context.Background(), game, &domain.Mod{ID: "1", SourceID: "src", Version: "1.0", GameID: "g1"}, "default")) + require.NoError(t, svc.SetModDeployed("src", "1", "g1", "default", true), + "seed Deployed=true so the post-disable assertion below actually proves a transition") // Corrupt the deployed file into a plain file (not a symlink) so the // symlink linker's Undeploy fails deterministically ("not a symlink"). From b2dcec26c1bf752b8fc053ef9ec1ff4ccfd5c39e Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 14:53:27 -0400 Subject: [PATCH 39/96] fix: self-heal stale deployed flag on already-disabled mods (#183) DisableMod's early return for an already-disabled mod skipped the SetModDeployed(false) call, so a mod stuck at enabled=false, deployed=true (e.g. one disabled before the #183 fix shipped) never converged - disable wasn't idempotent on the flag. The already-disabled path now clears a stale deployed=true too, non-fatally (same Note convention as the rest of #183), before returning. --- internal/core/flows.go | 19 +++++++++++++++++-- internal/core/flows_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/internal/core/flows.go b/internal/core/flows.go index c73d6a3..496afab 100644 --- a/internal/core/flows.go +++ b/internal/core/flows.go @@ -94,7 +94,10 @@ func (s *Service) EnableMod(ctx context.Context, game *domain.Game, profileName, // entry is kept so the mod can be re-enabled later without downloading again // — and marks it disabled (and not-deployed, #183) in the database. Returns // a result with Changed false — not an error — if the mod was already -// disabled. +// disabled. That already-disabled path still self-heals a stale +// deployed=true left behind by a pre-#183 disable (or any other drift): +// it clears the flag, non-fatally, before returning, so calling disable +// again converges deployed state even when enabled was already false. // // Undeploy failures are treated as non-fatal: the game files may already // have been removed manually, and refusing to record the user's intent to @@ -122,7 +125,19 @@ func (s *Service) DisableMod(ctx context.Context, game *domain.Game, profileName } if !mod.Enabled { - return &DisableResult{}, nil + // Self-heal (#183): a mod disabled before this fix shipped can be + // stuck with enabled=false but deployed=true forever, since nothing + // else clears the flag once the mod is already disabled. Clear it + // here too, under the same non-fatal Note convention as the + // already-enabled path below, so disable converges the flag even + // when called on an already-disabled mod. + result := &DisableResult{} + if mod.Deployed { + if err := s.SetModDeployed(sourceID, modID, game.ID, profileName, false); err != nil { + result.Notes = append(result.Notes, fmt.Sprintf("Warning: could not mark as not deployed: %v", err)) + } + } + return result, nil } result := &DisableResult{} diff --git a/internal/core/flows_test.go b/internal/core/flows_test.go index 048b48f..d2239f4 100644 --- a/internal/core/flows_test.go +++ b/internal/core/flows_test.go @@ -254,6 +254,34 @@ func TestService_DisableMod_AlreadyDisabledIsNoop(t *testing.T) { assert.False(t, result.Changed) } +// TestService_DisableMod_AlreadyDisabledSelfHealsStaleDeployedFlag pins +// #183's self-heal follow-up: a mod disabled before the #183 fix shipped +// (or otherwise drifted) can be stuck at enabled=false, deployed=true +// forever, since nothing else clears the flag once a mod is already +// disabled. Calling DisableMod again on it must converge deployed to +// false, still succeed, and still report Changed=false (the enabled +// status itself didn't change). +func TestService_DisableMod_AlreadyDisabledSelfHealsStaleDeployedFlag(t *testing.T) { + svc := newFlowsTestService(t) + gameDir := t.TempDir() + game := &domain.Game{ID: "g1", Name: "Game", ModPath: gameDir, LinkMethod: domain.LinkSymlink} + + seedInstalledMod(t, svc, game, "src", "1", "1.0", false, map[string][]byte{ + "plugin.esp": []byte("data"), + }) + require.NoError(t, svc.SetModDeployed("src", "1", "g1", "default", true), + "simulate a pre-#183 disable that left the stale deployed=true flag behind") + + result, err := svc.DisableMod(context.Background(), game, "default", "src", "1") + require.NoError(t, err, "self-healing the stale flag must still report success") + require.NotNil(t, result) + assert.False(t, result.Changed, "enabled status itself didn't change") + + mod, err := svc.GetInstalledMod("src", "1", "g1", "default") + require.NoError(t, err) + assert.False(t, mod.Deployed, "#183: re-disabling an already-disabled mod must self-heal a stale deployed=true") +} + func TestService_DisableMod_UnknownModReturnsErrModNotFound(t *testing.T) { svc := newFlowsTestService(t) game := &domain.Game{ID: "g1", Name: "Game", ModPath: t.TempDir(), LinkMethod: domain.LinkSymlink} From 59d80c065cd68e32dbeee11e2d56e170b5f274cb Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 15:09:12 -0400 Subject: [PATCH 40/96] feat: add {category}/{tags} placeholders to custom api search endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Custom api sources' search endpoint templates can now reference {category} and {tags}, fed from SearchQuery.Category/.Tags (URL-escaped; multiple tags comma-joined). Previously these fields were silently dropped for api-type custom sources. A definition whose search path omits the new placeholders is unaffected — the values are computed but never substituted in. Closes #120 Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 4 ++ README.md | 2 + internal/source/custom/api.go | 6 ++- internal/source/custom/api_test.go | 66 ++++++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 703923d..24dab99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Custom `api` sources' `search` endpoint gains `{category}`/`{tags}` path placeholders, fed from `SearchQuery.Category`/`.Tags` (URL-escaped; multiple tags comma-joined) — previously these were silently dropped with no way for a declarative source to express category/tag filtering. A definition whose `search` path omits the new placeholders is unaffected: the values are computed but never substituted in, matching today's behavior exactly (#120) + ## [1.27.1] - 2026-07-30 ### Fixed diff --git a/README.md b/README.md index 1d72743..26cd661 100644 --- a/README.md +++ b/README.md @@ -655,6 +655,8 @@ api: | `{page}` | The internal 0-based page number, plus `page_start` (default `1`) | `search` | | `{page_size}` | The requested page size (defaults to 20 when unspecified or ≤ 0) | `search` | | `{offset}` | The internal 0-based page × `page_size` — independent of `page_start`, for offset-paginated APIs | `search` | +| `{category}` | The search query's category filter (source-specific ID or name), empty when unset | `search` | +| `{tags}` | The search query's tag filters, comma-joined, empty when unset | `search` | | `{mod_id}` | The mod ID | `get_mod`, `mod_files`, `download_url` | | `{file_id}` | The file ID | `download_url` | diff --git a/internal/source/custom/api.go b/internal/source/custom/api.go index ff7c34a..b03dbfd 100644 --- a/internal/source/custom/api.go +++ b/internal/source/custom/api.go @@ -199,7 +199,9 @@ var ( // Search implements source.ModSource by executing the search endpoint // template and mapping the results (design §4). An undefined search endpoint -// is an unsupported capability. +// is an unsupported capability. {category} and {tags} (comma-joined) feed +// from query.Category/.Tags (#120); a template that omits them behaves +// exactly as before — the values are computed but never substituted in. func (a *API) Search(ctx context.Context, query source.SearchQuery) (source.SearchResult, error) { ep := a.endpoints.Search if ep == nil { @@ -221,6 +223,8 @@ func (a *API) Search(ctx context.Context, query source.SearchQuery) (source.Sear "page": strconv.Itoa(page + a.pageStart), "page_size": strconv.Itoa(pageSize), "offset": strconv.Itoa(page * pageSize), + "category": query.Category, + "tags": strings.Join(query.Tags, ","), } doc, err := a.getJSON(ctx, a.baseURL+buildEndpointURL(ep.Path, vals)) diff --git a/internal/source/custom/api_test.go b/internal/source/custom/api_test.go index 8cbc25a..a3aa644 100644 --- a/internal/source/custom/api_test.go +++ b/internal/source/custom/api_test.go @@ -459,3 +459,69 @@ func TestAPISearchNegativePageClamped(t *testing.T) { assert.Contains(t, gotPath, "page=1") assert.Contains(t, gotPath, "skip=0") } + +// TestAPISearchCategoryAndTagsPlaceholders pins {category}/{tags} substitution +// (#120): tags join with a comma (the issue leaves the delimiter open; comma +// is the documented convention), both are URL-escaped, and an endpoint whose +// template omits the placeholders is completely unaffected — the values are +// computed but never appear in the request, matching today's behavior for +// existing definitions. +func TestAPISearchCategoryAndTagsPlaceholders(t *testing.T) { + tests := []struct { + name string + path string + query source.SearchQuery + wantPath string + }{ + { + name: "category set", + path: "/mods?q={query}&category={category}", + query: source.SearchQuery{Query: "x", Category: "armor"}, + wantPath: "/mods?q=x&category=armor", + }, + { + name: "category unset substitutes empty", + path: "/mods?q={query}&category={category}", + query: source.SearchQuery{Query: "x"}, + wantPath: "/mods?q=x&category=", + }, + { + name: "multi-tag joins with comma and escapes", + path: "/mods?q={query}&tags={tags}", + query: source.SearchQuery{Query: "x", Tags: []string{"quality of life", "combat"}}, + wantPath: "/mods?q=x&tags=quality+of+life%2Ccombat", + }, + { + name: "tags unset substitutes empty", + path: "/mods?q={query}&tags={tags}", + query: source.SearchQuery{Query: "x"}, + wantPath: "/mods?q=x&tags=", + }, + { + name: "endpoint without placeholders is unchanged", + path: "/mods?q={query}&page={page}", + query: source.SearchQuery{Query: "x", Category: "armor", Tags: []string{"a", "b"}}, + wantPath: "/mods?q=x&page=1", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.String() + _, _ = w.Write([]byte(`{"results": []}`)) + })) + defer srv.Close() + + def := apiDef(srv.URL) + def.API.Endpoints.Search = &EndpointConfig{Path: tt.path, List: "results"} + a, err := NewAPI(def) + require.NoError(t, err) + + _, err = a.Search(context.Background(), tt.query) + require.NoError(t, err) + assert.Equal(t, tt.wantPath, gotPath) + }) + } +} From 1f4c2ce8c889da838c5017d453dbfb9b5818c9a5 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 15:13:12 -0400 Subject: [PATCH 41/96] fix: fail loud on unknown link_method/deploy_mode values (#172) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit domain.ParseLinkMethod and domain.ParseDeployMode now return (value, ok) instead of silently mapping any unrecognized string to their default. Empty string still keeps today's default (backward compatible). Every load site that parses a raw string into one of these enums now checks ok and returns a load-time error naming the field, offending value, owning game/profile, and valid options: config.Load (default_link_method), LoadGames (link_method/deploy_mode per game), LoadProfile (profile-level link_method), and ImportProfile (imported profile's link_method). Tests updated: TestParseLinkMethod/TestParseDeployMode's "unknown/mismatched case defaults to X" cases are inverted to "is rejected" (ok=false) — this is a deliberate contract change, not incidental drift. New load-layer tests pin the fail-loud behavior for games.yaml, config.yaml, profile files, and imported profiles. Scope note: internal/core/service.go's GetEffectiveLinkMethod still swallows any LoadProfile error (including the new validation error) by pre-existing design to degrade gracefully on a missing/unreadable profile. Fixing that would mean changing its signature and ~10 call sites across core/flows.go and cmd/lmm — out of scope here per coordinator decision; tracked as a follow-up. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 4 + docs/configuration.md | 2 + internal/domain/errors.go | 23 ++++-- internal/domain/game.go | 29 ++++--- internal/domain/game_test.go | 57 ++++++++------ internal/storage/config/config.go | 7 +- internal/storage/config/config_test.go | 98 ++++++++++++++++++++++++ internal/storage/config/games.go | 14 +++- internal/storage/config/profiles.go | 16 +++- internal/storage/config/profiles_test.go | 15 ++++ 10 files changed, 222 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 703923d..b9d0401 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- An unrecognized, non-empty `link_method` (`games.yaml`, profile files, imported profiles) or `deploy_mode` (`games.yaml`) is now a load-time error naming the field, the offending value, the owning game/profile, and the valid options — instead of silently falling back to the default (`symlink`/`extract`). **Breaking for configs that were already silently misbehaving:** a typo like `deploy_mode: compil` previously ran as `extract` with no warning; it now refuses to load until fixed. An empty/absent value is unaffected and keeps today's default exactly (#172) + ## [1.27.1] - 2026-07-30 ### Fixed diff --git a/docs/configuration.md b/docs/configuration.md index eb03632..eddd97e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -2,6 +2,8 @@ lmm uses YAML configuration files under `~/.config/lmm/` (or the directory set with `--config`). +`link_method` and `deploy_mode` fields (in `config.yaml`, `games.yaml`, and profile files) are validated at load time: leaving one unset keeps its documented default, but a value that doesn't exactly match one of the listed options — a typo like `deploy_mode: compil` — is a load-time error naming the field, the offending value, and the valid options, not a silent fallback. + ## config.yaml Global application settings. Optional; defaults apply if the file is missing. diff --git a/internal/domain/errors.go b/internal/domain/errors.go index 1804820..a4c1137 100644 --- a/internal/domain/errors.go +++ b/internal/domain/errors.go @@ -18,13 +18,22 @@ var ( // ErrInvalidGameID applies the same rule to game IDs, which are joined // into the same on-disk paths (and reachable from untrusted YAML via // profile import). - ErrInvalidGameID = errors.New("invalid game ID") - ErrDependencyLoop = errors.New("circular dependency detected") - ErrAuthRequired = errors.New("authentication required") - ErrInvalidConfig = errors.New("invalid configuration") - ErrFileConflict = errors.New("file conflict detected") - ErrDownloadFailed = errors.New("download failed") - ErrLinkFailed = errors.New("link operation failed") + ErrInvalidGameID = errors.New("invalid game ID") + // ErrInvalidLinkMethod flags a link_method value that is neither empty + // (which keeps the existing default) nor one of the recognized names + // (symlink, hardlink, copy). Config loaders wrap it with the offending + // field, value, and owning game/profile so the message names exactly + // what's wrong and how to fix it (#172). + ErrInvalidLinkMethod = errors.New("invalid link method") + // ErrInvalidDeployMode is ErrInvalidLinkMethod's counterpart for + // deploy_mode (extract, copy) (#172). + ErrInvalidDeployMode = errors.New("invalid deploy mode") + ErrDependencyLoop = errors.New("circular dependency detected") + ErrAuthRequired = errors.New("authentication required") + ErrInvalidConfig = errors.New("invalid configuration") + ErrFileConflict = errors.New("file conflict detected") + ErrDownloadFailed = errors.New("download failed") + ErrLinkFailed = errors.New("link operation failed") ) // DeployError aggregates a primary failure with optional rollback / cleanup diff --git a/internal/domain/game.go b/internal/domain/game.go index 871ca03..6ce916b 100644 --- a/internal/domain/game.go +++ b/internal/domain/game.go @@ -22,15 +22,22 @@ func (m LinkMethod) String() string { } } -// ParseLinkMethod converts a string to LinkMethod -func ParseLinkMethod(s string) LinkMethod { +// ParseLinkMethod converts a string to LinkMethod. An empty string is not +// yet set and returns the default (symlink) with ok=true, so configs that +// never set link_method keep working unchanged. Any other unrecognized +// string returns ok=false so the caller can fail loud (naming the field, +// offending value, and owning game/profile) instead of silently defaulting +// (#172). +func ParseLinkMethod(s string) (method LinkMethod, ok bool) { switch s { + case "", "symlink": + return LinkSymlink, true case "hardlink": - return LinkHardlink + return LinkHardlink, true case "copy": - return LinkCopy + return LinkCopy, true default: - return LinkSymlink + return LinkSymlink, false } } @@ -67,12 +74,16 @@ func (m DeployMode) String() string { } } -// ParseDeployMode converts a string to DeployMode -func ParseDeployMode(s string) DeployMode { +// ParseDeployMode converts a string to DeployMode. Mirrors ParseLinkMethod's +// fail-loud contract: empty keeps the default (extract) with ok=true; any +// other unrecognized string returns ok=false (#172). +func ParseDeployMode(s string) (mode DeployMode, ok bool) { switch s { + case "", "extract": + return DeployExtract, true case "copy": - return DeployCopy + return DeployCopy, true default: - return DeployExtract + return DeployExtract, false } } diff --git a/internal/domain/game_test.go b/internal/domain/game_test.go index f170718..2410080 100644 --- a/internal/domain/game_test.go +++ b/internal/domain/game_test.go @@ -25,27 +25,34 @@ func TestLinkMethod_String(t *testing.T) { } } +// TestParseLinkMethod pins the fail-loud contract from #172: empty keeps +// today's default, everything else must be an exact recognized name or the +// parse is rejected (ok=false) instead of silently defaulting. Inverted from +// the pre-#172 version of this test, which asserted "bogus"/mismatched-case +// input silently fell back to LinkSymlink. func TestParseLinkMethod(t *testing.T) { tests := []struct { - name string - input string - want LinkMethod + name string + input string + wantMode LinkMethod + wantOK bool }{ - {"hardlink", "hardlink", LinkHardlink}, - {"copy", "copy", LinkCopy}, - {"symlink explicit", "symlink", LinkSymlink}, - {"empty defaults to symlink", "", LinkSymlink}, - {"unknown defaults to symlink", "bogus", LinkSymlink}, + {"hardlink", "hardlink", LinkHardlink, true}, + {"copy", "copy", LinkCopy, true}, + {"symlink explicit", "symlink", LinkSymlink, true}, + {"empty defaults to symlink", "", LinkSymlink, true}, + {"unknown is rejected", "bogus", LinkSymlink, false}, // ParseLinkMethod compares against exact lowercase literals, so any - // other casing falls through to the default rather than being - // case-normalized. - {"case sensitive - not matched", "Hardlink", LinkSymlink}, - {"case sensitive - upper not matched", "COPY", LinkSymlink}, + // other casing is rejected rather than being case-normalized. + {"case sensitive - rejected", "Hardlink", LinkSymlink, false}, + {"case sensitive - upper rejected", "COPY", LinkSymlink, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, ParseLinkMethod(tt.input)) + got, ok := ParseLinkMethod(tt.input) + assert.Equal(t, tt.wantMode, got) + assert.Equal(t, tt.wantOK, ok) }) } } @@ -70,23 +77,29 @@ func TestDeployMode_String(t *testing.T) { } } +// TestParseDeployMode mirrors TestParseLinkMethod's fail-loud contract. +// Inverted from the pre-#172 version, which asserted "bogus"/mismatched-case +// input silently fell back to DeployExtract. func TestParseDeployMode(t *testing.T) { tests := []struct { - name string - input string - want DeployMode + name string + input string + wantMode DeployMode + wantOK bool }{ - {"copy", "copy", DeployCopy}, - {"extract explicit", "extract", DeployExtract}, - {"empty defaults to extract", "", DeployExtract}, - {"unknown defaults to extract", "bogus", DeployExtract}, + {"copy", "copy", DeployCopy, true}, + {"extract explicit", "extract", DeployExtract, true}, + {"empty defaults to extract", "", DeployExtract, true}, + {"unknown is rejected", "bogus", DeployExtract, false}, // Same exact-match, no-case-folding behavior as ParseLinkMethod. - {"case sensitive - not matched", "Copy", DeployExtract}, + {"case sensitive - rejected", "Copy", DeployExtract, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, ParseDeployMode(tt.input)) + got, ok := ParseDeployMode(tt.input) + assert.Equal(t, tt.wantMode, got) + assert.Equal(t, tt.wantOK, ok) }) } } diff --git a/internal/storage/config/config.go b/internal/storage/config/config.go index 52d2aa8..06d91db 100644 --- a/internal/storage/config/config.go +++ b/internal/storage/config/config.go @@ -44,7 +44,12 @@ func Load(configDir string) (*Config, error) { // Convert string to LinkMethod if cfg.LinkMethodStr != "" { - cfg.DefaultLinkMethod = domain.ParseLinkMethod(cfg.LinkMethodStr) + method, ok := domain.ParseLinkMethod(cfg.LinkMethodStr) + if !ok { + return nil, fmt.Errorf("%w: config.yaml: default_link_method %q (valid: symlink, hardlink, copy)", + domain.ErrInvalidLinkMethod, cfg.LinkMethodStr) + } + cfg.DefaultLinkMethod = method } // Expand ~ in cache path diff --git a/internal/storage/config/config_test.go b/internal/storage/config/config_test.go index fa0a6e7..04c29d0 100644 --- a/internal/storage/config/config_test.go +++ b/internal/storage/config/config_test.go @@ -39,6 +39,28 @@ keybindings: standard assert.Equal(t, "standard", cfg.Keybindings) } +// TestLoadConfig_RejectsUnknownDefaultLinkMethod pins #172's fail-loud +// contract: a non-empty, unrecognized default_link_method is a load-time +// error naming the field, offending value, and valid options, instead of +// silently defaulting to symlink. +func TestLoadConfig_RejectsUnknownDefaultLinkMethod(t *testing.T) { + dir := t.TempDir() + configPath := filepath.Join(dir, "config.yaml") + + content := "default_link_method: bogus\n" + require.NoError(t, os.WriteFile(configPath, []byte(content), 0644)) + + _, err := config.Load(dir) + + require.Error(t, err) + assert.ErrorIs(t, err, domain.ErrInvalidLinkMethod) + assert.Contains(t, err.Error(), "default_link_method") + assert.Contains(t, err.Error(), "bogus") + assert.Contains(t, err.Error(), "symlink") + assert.Contains(t, err.Error(), "hardlink") + assert.Contains(t, err.Error(), "copy") +} + func TestLoadGames_Empty(t *testing.T) { dir := t.TempDir() games, err := config.LoadGames(dir) @@ -74,6 +96,60 @@ games: assert.Equal(t, "skyrimspecialedition", game.SourceIDs["nexusmods"]) } +// TestLoadGames_RejectsUnknownLinkMethod and TestLoadGames_RejectsUnknownDeployMode +// pin #172's fail-loud contract for games.yaml: a non-empty, unrecognized +// value is a load-time error naming the field, offending value, the game ID, +// and valid options, instead of silently defaulting. +func TestLoadGames_RejectsUnknownLinkMethod(t *testing.T) { + dir := t.TempDir() + gamesPath := filepath.Join(dir, "games.yaml") + + content := ` +games: + skyrim-se: + name: Skyrim Special Edition + install_path: /games/skyrim + mod_path: /games/skyrim/Data + sources: + nexusmods: skyrimspecialedition + link_method: bogus +` + require.NoError(t, os.WriteFile(gamesPath, []byte(content), 0644)) + + _, err := config.LoadGames(dir) + + require.Error(t, err) + assert.ErrorIs(t, err, domain.ErrInvalidLinkMethod) + assert.Contains(t, err.Error(), "skyrim-se") + assert.Contains(t, err.Error(), "link_method") + assert.Contains(t, err.Error(), "bogus") +} + +func TestLoadGames_RejectsUnknownDeployMode(t *testing.T) { + dir := t.TempDir() + gamesPath := filepath.Join(dir, "games.yaml") + + content := ` +games: + skyrim-se: + name: Skyrim Special Edition + install_path: /games/skyrim + mod_path: /games/skyrim/Data + sources: + nexusmods: skyrimspecialedition + deploy_mode: compil +` + require.NoError(t, os.WriteFile(gamesPath, []byte(content), 0644)) + + _, err := config.LoadGames(dir) + + require.Error(t, err) + assert.ErrorIs(t, err, domain.ErrInvalidDeployMode) + assert.Contains(t, err.Error(), "skyrim-se") + assert.Contains(t, err.Error(), "deploy_mode") + assert.Contains(t, err.Error(), "compil") +} + func TestSaveGame(t *testing.T) { dir := t.TempDir() @@ -129,6 +205,28 @@ link_method: symlink assert.Equal(t, "12345", profile.Mods[0].ModID) } +// TestLoadProfile_RejectsUnknownLinkMethod pins #172's fail-loud contract +// for profile-level link_method: a non-empty, unrecognized value is a +// load-time error naming the field, offending value, the profile/game, and +// valid options. +func TestLoadProfile_RejectsUnknownLinkMethod(t *testing.T) { + dir := t.TempDir() + profileDir := filepath.Join(dir, "games", "skyrim-se", "profiles") + require.NoError(t, os.MkdirAll(profileDir, 0755)) + + content := "name: default\ngame_id: skyrim-se\nlink_method: bogus\n" + require.NoError(t, os.WriteFile(filepath.Join(profileDir, "default.yaml"), []byte(content), 0644)) + + _, err := config.LoadProfile(dir, "skyrim-se", "default") + + require.Error(t, err) + assert.ErrorIs(t, err, domain.ErrInvalidLinkMethod) + assert.Contains(t, err.Error(), "default") + assert.Contains(t, err.Error(), "skyrim-se") + assert.Contains(t, err.Error(), "link_method") + assert.Contains(t, err.Error(), "bogus") +} + func TestSaveProfile(t *testing.T) { dir := t.TempDir() diff --git a/internal/storage/config/games.go b/internal/storage/config/games.go index ee7d142..be30e93 100644 --- a/internal/storage/config/games.go +++ b/internal/storage/config/games.go @@ -87,16 +87,26 @@ func loadGamesLocked(configDir string) (map[string]*domain.Game, error) { } games := make(map[string]*domain.Game) for id, cfg := range gamesFile.Games { + linkMethod, ok := domain.ParseLinkMethod(cfg.LinkMethod) + if !ok { + return nil, fmt.Errorf("%w: games.yaml: game %q: link_method %q (valid: symlink, hardlink, copy)", + domain.ErrInvalidLinkMethod, id, cfg.LinkMethod) + } + deployMode, ok := domain.ParseDeployMode(cfg.DeployMode) + if !ok { + return nil, fmt.Errorf("%w: games.yaml: game %q: deploy_mode %q (valid: extract, copy)", + domain.ErrInvalidDeployMode, id, cfg.DeployMode) + } games[id] = &domain.Game{ ID: id, Name: cfg.Name, InstallPath: ExpandPath(cfg.InstallPath), ModPath: ExpandPath(cfg.ModPath), SourceIDs: cfg.Sources, - LinkMethod: domain.ParseLinkMethod(cfg.LinkMethod), + LinkMethod: linkMethod, LinkMethodExplicit: cfg.LinkMethod != "", CachePath: ExpandPath(cfg.CachePath), - DeployMode: domain.ParseDeployMode(cfg.DeployMode), + DeployMode: deployMode, Hooks: domain.GameHooks{ Install: domain.HookConfig{ BeforeAll: ExpandPath(cfg.Hooks.Install.BeforeAll), diff --git a/internal/storage/config/profiles.go b/internal/storage/config/profiles.go index cdac883..9604a7e 100644 --- a/internal/storage/config/profiles.go +++ b/internal/storage/config/profiles.go @@ -135,10 +135,16 @@ func LoadProfile(configDir, gameID, profileName string) (*domain.Profile, error) return nil, fmt.Errorf("parsing profile: %w", err) } + linkMethod, ok := domain.ParseLinkMethod(cfg.LinkMethod) + if !ok { + return nil, fmt.Errorf("%w: profile %q (game %q): link_method %q (valid: symlink, hardlink, copy)", + domain.ErrInvalidLinkMethod, profileName, gameID, cfg.LinkMethod) + } + profile := &domain.Profile{ Name: cfg.Name, GameID: cfg.GameID, - LinkMethod: domain.ParseLinkMethod(cfg.LinkMethod), + LinkMethod: linkMethod, LinkMethodExplicit: cfg.LinkMethod != "", IsDefault: cfg.IsDefault, Mods: make([]domain.ModReference, len(cfg.Mods)), @@ -294,11 +300,17 @@ func ImportProfile(data []byte) (*domain.Profile, error) { return nil, fmt.Errorf("parsing exported profile: %w", err) } + linkMethod, ok := domain.ParseLinkMethod(exported.LinkMethod) + if !ok { + return nil, fmt.Errorf("%w: imported profile %q (game %q): link_method %q (valid: symlink, hardlink, copy)", + domain.ErrInvalidLinkMethod, exported.Name, exported.GameID, exported.LinkMethod) + } + p := &domain.Profile{ Name: exported.Name, GameID: exported.GameID, Mods: exported.Mods, - LinkMethod: domain.ParseLinkMethod(exported.LinkMethod), + LinkMethod: linkMethod, LinkMethodExplicit: exported.LinkMethod != "", } if len(exported.Overrides) > 0 { diff --git a/internal/storage/config/profiles_test.go b/internal/storage/config/profiles_test.go index 824dfcd..b8944ed 100644 --- a/internal/storage/config/profiles_test.go +++ b/internal/storage/config/profiles_test.go @@ -349,6 +349,21 @@ func TestImportProfile_TracksLinkMethodExplicit(t *testing.T) { assert.Equal(t, domain.LinkCopy, imported.LinkMethod) } +// TestImportProfile_RejectsUnknownLinkMethod pins #172's fail-loud contract +// for imported profiles: a non-empty, unrecognized link_method is a +// load-time error naming the field, offending value, the profile/game, and +// valid options, instead of silently defaulting. +func TestImportProfile_RejectsUnknownLinkMethod(t *testing.T) { + _, err := ImportProfile([]byte("name: default\ngame_id: skyrim-se\nlink_method: bogus\n")) + + require.Error(t, err) + assert.ErrorIs(t, err, domain.ErrInvalidLinkMethod) + assert.Contains(t, err.Error(), "default") + assert.Contains(t, err.Error(), "skyrim-se") + assert.Contains(t, err.Error(), "link_method") + assert.Contains(t, err.Error(), "bogus") +} + // TestSaveProfile_PreservesLockedMarker guards that Locked is written to YAML when true, // omitted when false (omitempty), verified against the raw saved YAML // (LoadProfile round-trip behavior is covered by the sibling Load tests). From f720ce29095a750e8b5859ca81ef64ba86582cb1 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 15:16:16 -0400 Subject: [PATCH 42/96] fix: route local .exmodz imports through the compile pipeline (#173) lmm import copied/extracted .exmodz archives as-is for a deploy_mode: compile game, landing an uncompiled diff in the cache instead of a deployable _P.pak. Import now mirrors DownloadModToCache's DeployCompile branch: when the archive is .exmodz-eligible, it resolves the game's mapped Compiler-capable source from the registry (no per-archive source is pinned at import time the way a download has one), compiles against the installed base pak, and caches the result under the same _P.pak naming DownloadModToCache uses. Missing compiler source or missing base pak fail loud instead of silently caching the raw archive; non-.exmodz imports are untouched. --- CHANGELOG.md | 1 + internal/core/importer.go | 67 ++++- internal/core/service.go | 32 ++ internal/core/service_import_compile_test.go | 299 +++++++++++++++++++ 4 files changed, 398 insertions(+), 1 deletion(-) create mode 100644 internal/core/service_import_compile_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 4aee183..ec5b839 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- `lmm import` of a local `.exmodz` file for a `deploy_mode: compile` game (Icarus) now routes through the same compile step as a download: it resolves the game's mapped `source.Compiler`-capable source from the registry and compiles the archive against the installed base pak, caching the resulting `_P.pak` the same way `DownloadModToCache` does. Previously the import path extracted/copied `.exmodz` files as-is, landing an uncompiled archive in the cache instead of a deployable pak. A missing compiler-capable source or missing base pak now fails loud with an actionable error rather than silently caching the uncompiled file; non-`.exmodz` imports are unaffected (#173) - `lmm mod disable` undeployed a mod's files and cleared `enabled`, but never cleared `deployed` — `lmm list -v` kept showing DEPLOYED yes after disable. The disable flow now clears `deployed` unconditionally after the undeploy attempt, even when the undeploy itself only partially succeeds (already a non-fatal, Note-reported condition), so the flag always reflects disable-intent rather than lagging behind a best-effort file cleanup. The symmetric enable path had the same gap — enabling a disabled mod re-deployed its files without ever setting `deployed` back to true — and is fixed the same way. Both `SetModDeployed` calls follow the same non-fatal Note convention already used by `DeployProfile`/`PurgeProfile` for this same setter: a failure to record the flag doesn't block the primary enable/disable outcome (#183) ## [1.27.1] - 2026-07-30 diff --git a/internal/core/importer.go b/internal/core/importer.go index bf264f5..cd28300 100644 --- a/internal/core/importer.go +++ b/internal/core/importer.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/source" "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" "github.com/google/uuid" ) @@ -36,6 +37,15 @@ type Importer struct { // stagingRoot is where archives are extracted before being committed to the // cache. Empty means fall back to $TMPDIR — see newStagingDir. stagingRoot string + // resolveCompiler resolves the Compiler-capable source mapped to a + // DeployCompile game's registry entry (#173), consulted only when + // importing a ".exmodz" archive for such a game — Import has no + // per-archive source pinned the way DownloadModToCache does, so it must + // look up the game's configured sources instead. nil when the Importer + // was built via the standalone NewImporter (no Service context): + // compiling an .exmodz through such an Importer fails loud rather than + // silently caching the uncompiled archive. + resolveCompiler func(gameID string) (source.Compiler, error) } // NewImporter creates a new Importer that stages extraction in the OS temp dir. @@ -52,6 +62,7 @@ func NewImporter(cache *cache.Cache) *Importer { func (s *Service) NewImporter(game *domain.Game) *Importer { imp := NewImporter(s.GetGameCache(game)) imp.stagingRoot = s.stagingRoot() + imp.resolveCompiler = s.compilerSourceForGame return imp } @@ -96,7 +107,61 @@ func (i *Importer) Import(ctx context.Context, archivePath string, game *domain. var fileCount int // Handle based on game's deploy mode - if game.DeployMode == domain.DeployCopy { + if game.DeployMode == domain.DeployCompile && isExmodzFile(filename) { + // Compile mode (#173): mirror Service.DownloadModToCache's + // DeployCompile branch — compile the archive against the game's + // installed base pak instead of extracting or copying it verbatim, + // caching the compiled *_P.pak the same way a downloaded .exmodz + // would be. Unlike the download path, Import has no per-archive + // source pinned to check for source.Compiler, so it resolves the + // game's mapped compiler-capable source from the registry instead. + if i.resolveCompiler == nil { + return nil, fmt.Errorf("game %q requires DeployCompile to import %q, but no compiler-capable source is configured for this game", game.ID, filename) + } + compiler, err := i.resolveCompiler(game.ID) + if err != nil { + return nil, err + } + basePakPath, err := resolveBasePak(game) + if err != nil { + return nil, err + } + + modName = strings.TrimSuffix(filename, filepath.Ext(filename)) + if version != "" && version != "unknown" { + if idx := strings.LastIndex(modName, version); idx > 0 { + modName = strings.TrimRight(modName[:idx], "-_ ") + } + } + + // Compile into a staging dir first so a mid-compile failure never + // leaves a partial/uncompiled artifact in the cache (mirrors #136 + // review's fix for the download path's compile branch). + tempDir, err := newStagingDir(i.stagingRoot, "lmm-import-compile-*") + if err != nil { + return nil, err + } + defer os.RemoveAll(tempDir) //nolint:errcheck + + destName := compiledFileName(filename) + compiledPath := filepath.Join(tempDir, destName) + if err := compiler.Compile(ctx, basePakPath, archivePath, compiledPath); err != nil { + return nil, fmt.Errorf("compiling mod: %w", err) + } + + cachePath := i.cache.ModPath(game.ID, sourceID, modID, version) + // Remove existing cache if present (re-import case) + if err := os.RemoveAll(cachePath); err != nil { + return nil, fmt.Errorf("removing existing cache for re-import: %w", err) + } + if err := os.MkdirAll(cachePath, 0755); err != nil { + return nil, fmt.Errorf("creating cache directory: %w", err) + } + if err := copyFileStreaming(compiledPath, filepath.Join(cachePath, destName)); err != nil { + return nil, fmt.Errorf("moving compiled mod to cache: %w", err) + } + fileCount = 1 + } else if game.DeployMode == domain.DeployCopy { // Copy mode: just copy the file as-is to cache (don't extract) modName = strings.TrimSuffix(filename, filepath.Ext(filename)) if version != "" && version != "unknown" { diff --git a/internal/core/service.go b/internal/core/service.go index 2bdd2e4..8fab764 100644 --- a/internal/core/service.go +++ b/internal/core/service.go @@ -165,6 +165,38 @@ func (s *Service) SourcesForGame(gameID string) ([]source.ModSource, error) { return srcs, nil } +// compilerSourceForGame resolves the sole Compiler-capable source +// registered for gameID (#173). The download path pins its Compiler check +// to the specific source a file was downloaded from (DownloadModToCache's +// src.(source.Compiler) check); Importer.Import has no such per-archive +// source to key off of, so it resolves against every source the game maps +// in its registry instead — matching resolveBasePak's v1 scope of "Icarus +// only", at most one of a game's configured sources implements Compiler +// today. Zero is the expected failure when the game (or its Compiler +// source) isn't configured; more than one is treated as ambiguous rather +// than picking arbitrarily — both fail loud instead of letting an .exmodz +// import silently skip compilation. +func (s *Service) compilerSourceForGame(gameID string) (source.Compiler, error) { + srcs, err := s.SourcesForGame(gameID) + if err != nil { + return nil, err + } + var compilers []source.Compiler + for _, src := range srcs { + if c, ok := src.(source.Compiler); ok { + compilers = append(compilers, c) + } + } + switch len(compilers) { + case 0: + return nil, fmt.Errorf("game %q requires DeployCompile but has no compiler-capable source configured (map a source implementing source.Compiler in the game's sources)", gameID) + case 1: + return compilers[0], nil + default: + return nil, fmt.Errorf("game %q has multiple compiler-capable sources configured; ambiguous compile source", gameID) + } +} + // SourceWarning reports a per-source failure during an aggregate operation. type SourceWarning struct { SourceID string diff --git a/internal/core/service_import_compile_test.go b/internal/core/service_import_compile_test.go new file mode 100644 index 0000000..136375c --- /dev/null +++ b/internal/core/service_import_compile_test.go @@ -0,0 +1,299 @@ +package core_test + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/core" + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" + "github.com/stretchr/testify/require" +) + +// failingCompilerSource wraps fakeCompilerSource (defined in +// service_icarus_compile_test.go) and shadows Compile to always fail, +// letting failure-leg tests below prove a mid-compile error never leaves a +// partial artifact behind (#173, mirroring #136 review's "remove partial +// output pak on mid-compile failure" fix for the download path). +type failingCompilerSource struct { + *fakeCompilerSource +} + +func (s *failingCompilerSource) Compile(ctx context.Context, basePakPath, sourceFilePath, outputPath string) error { + return fmt.Errorf("boom: compile always fails") +} + +// newImportCompileTestGame builds a DeployCompile game with a registered, +// game-mapped compiler source and an installed base pak - the setup #173's +// import path needs to resolve a Compiler the same way +// Service.DownloadModToCache resolves one from the download's own source, +// except import has no per-download source pinned, so it must resolve the +// compiler from the game's registered sources instead (game.SourceIDs). +func newImportCompileTestGame(t *testing.T) (*core.Service, *fakeCompilerSource, *domain.Game) { + t.Helper() + + installDir := t.TempDir() + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + require.NoError(t, os.WriteFile(basePak, []byte("fake-base-pak"), 0o644)) + + cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} + svc, err := core.NewService(cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + src := &fakeCompilerSource{} + svc.RegisterSource(src) + + game := &domain.Game{ + ID: "icarus", + InstallPath: installDir, + ModPath: t.TempDir(), + DeployMode: domain.DeployCompile, + SourceIDs: map[string]string{"fake-compiler": "external-icarus-id"}, + } + require.NoError(t, svc.AddGame(game)) + + return svc, src, game +} + +func TestImportMod_DeployCompile_ExmodzCompiles(t *testing.T) { + svc, src, game := newImportCompileTestGame(t) + + tempDir := t.TempDir() + archivePath := filepath.Join(tempDir, "Bear_Mount.exmodz") + require.NoError(t, os.WriteFile(archivePath, []byte("fake-exmodz-bytes"), 0o644)) + + importer := svc.NewImporter(game) + result, err := importer.Import(context.Background(), archivePath, game, core.ImportOptions{}) + require.NoError(t, err) + require.Equal(t, 1, result.FilesExtracted) + require.Equal(t, 1, src.compileCalls) + + gameCache := svc.GetGameCache(game) + files, err := gameCache.ListFiles(game.ID, result.Mod.SourceID, result.Mod.ID, result.Mod.Version) + require.NoError(t, err) + require.Equal(t, []string{"Bear_Mount_P.pak"}, files) + + data, err := os.ReadFile(gameCache.GetFilePath(game.ID, result.Mod.SourceID, result.Mod.ID, result.Mod.Version, files[0])) + require.NoError(t, err) + require.Equal(t, "fake-exmodz-bytes", string(data)) +} + +// TestImportMod_DeployCompile_RoutesPerFile mirrors +// TestDownloadMod_DeployCompile_RoutesPerFile (service_icarus_compile_test.go): +// only a ".exmodz" suffix (case-insensitive) takes the compile branch. A +// plain ".pak" import is untouched by #173 - it falls through to the +// existing extract-mode branch exactly as it did before this change, which +// today means "unsupported archive format" (pak isn't a recognized +// archive), pinned here as a regression proof that non-exmodz import +// behavior is unchanged. +func TestImportMod_DeployCompile_RoutesPerFile(t *testing.T) { + tests := []struct { + name string + fileName string + wantCompiled bool + wantCachedName string + wantErrContains string + }{ + {name: "exmodz file takes the compile branch", fileName: "Bear_Mount.exmodz", wantCompiled: true, wantCachedName: "Bear_Mount_P.pak"}, + {name: "EXMODZ file takes the compile branch case-insensitively", fileName: "Bear_Mount.EXMODZ", wantCompiled: true, wantCachedName: "Bear_Mount_P.pak"}, + {name: "pak file is unaffected: today's unsupported-archive error is unchanged", fileName: "Bear_Mount.pak", wantErrContains: "unsupported archive format"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + svc, src, game := newImportCompileTestGame(t) + + tempDir := t.TempDir() + archivePath := filepath.Join(tempDir, tt.fileName) + require.NoError(t, os.WriteFile(archivePath, []byte("fake-bytes"), 0o644)) + + importer := svc.NewImporter(game) + result, err := importer.Import(context.Background(), archivePath, game, core.ImportOptions{}) + + if tt.wantErrContains != "" { + require.Error(t, err) + require.Contains(t, err.Error(), tt.wantErrContains) + require.Equal(t, 0, src.compileCalls) + return + } + + require.NoError(t, err) + wantCompileCalls := 0 + if tt.wantCompiled { + wantCompileCalls = 1 + } + require.Equal(t, wantCompileCalls, src.compileCalls) + + gameCache := svc.GetGameCache(game) + files, err := gameCache.ListFiles(game.ID, result.Mod.SourceID, result.Mod.ID, result.Mod.Version) + require.NoError(t, err) + require.Equal(t, []string{tt.wantCachedName}, files) + }) + } +} + +// TestImportMod_DeployCompile_ZipPassthroughUnaffected proves a regular +// (non-exmodz) archive import for a DeployCompile game is byte-for-byte +// identical to the same import against a DeployExtract game - #173 only +// inserts a new leading branch keyed on isExmodzFile, it must never change +// behavior for anything else. +func TestImportMod_DeployCompile_ZipPassthroughUnaffected(t *testing.T) { + makeArchive := func(t *testing.T) string { + t.Helper() + tempDir := t.TempDir() + archivePath := filepath.Join(tempDir, "SomeMod.zip") + createImportTestZip(t, archivePath, map[string]string{"plugin.txt": "test content"}) + return archivePath + } + + compileSvc, compileSrc, compileGame := newImportCompileTestGame(t) + compileImporter := compileSvc.NewImporter(compileGame) + compileResult, err := compileImporter.Import(context.Background(), makeArchive(t), compileGame, core.ImportOptions{}) + require.NoError(t, err) + require.Equal(t, 0, compileSrc.compileCalls) + + extractSvc, extractSrc, extractGame := newImportCompileTestGame(t) + extractGame.DeployMode = domain.DeployExtract + extractImporter := extractSvc.NewImporter(extractGame) + extractResult, err := extractImporter.Import(context.Background(), makeArchive(t), extractGame, core.ImportOptions{}) + require.NoError(t, err) + require.Equal(t, 0, extractSrc.compileCalls) + + require.Equal(t, extractResult.FilesExtracted, compileResult.FilesExtracted) + require.Equal(t, extractResult.Mod.Name, compileResult.Mod.Name) +} + +// TestImportMod_DeployCompile_NoCompilerSourceFailsLoud pins the "never +// silently cache an uncompiled .exmodz" requirement (#173): a DeployCompile +// game with no Compiler-capable source mapped in its SourceIDs must fail +// loud with an actionable error instead of falling through to extract/copy. +func TestImportMod_DeployCompile_NoCompilerSourceFailsLoud(t *testing.T) { + installDir := t.TempDir() + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + require.NoError(t, os.WriteFile(basePak, []byte("fake-base-pak"), 0o644)) + + cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} + svc, err := core.NewService(cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + // No RegisterSource call at all - the game has no source mapped, let + // alone a Compiler-capable one. + game := &domain.Game{ID: "icarus", InstallPath: installDir, ModPath: t.TempDir(), DeployMode: domain.DeployCompile} + require.NoError(t, svc.AddGame(game)) + + tempDir := t.TempDir() + archivePath := filepath.Join(tempDir, "Bear_Mount.exmodz") + require.NoError(t, os.WriteFile(archivePath, []byte("fake-exmodz-bytes"), 0o644)) + + importer := svc.NewImporter(game) + result, err := importer.Import(context.Background(), archivePath, game, core.ImportOptions{}) + require.Error(t, err) + require.Nil(t, result) + require.Contains(t, err.Error(), "compiler") + + _, statErr := os.Stat(filepath.Join(cfg.CacheDir, game.ID)) + require.True(t, os.IsNotExist(statErr), "no cache entry should have been created") +} + +// TestImportMod_DeployCompile_MissingBasePakFailsLoud pins the second +// "fail loud when compilation is impossible" leg (#173): a game whose +// installed base pak is missing must error instead of compiling against +// nothing or silently caching the raw archive. +func TestImportMod_DeployCompile_MissingBasePakFailsLoud(t *testing.T) { + installDir := t.TempDir() // no Icarus/Content/Data/data.pak written + + cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} + svc, err := core.NewService(cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + src := &fakeCompilerSource{} + svc.RegisterSource(src) + + game := &domain.Game{ + ID: "icarus", + InstallPath: installDir, + ModPath: t.TempDir(), + DeployMode: domain.DeployCompile, + SourceIDs: map[string]string{"fake-compiler": "external-icarus-id"}, + } + require.NoError(t, svc.AddGame(game)) + + tempDir := t.TempDir() + archivePath := filepath.Join(tempDir, "Bear_Mount.exmodz") + require.NoError(t, os.WriteFile(archivePath, []byte("fake-exmodz-bytes"), 0o644)) + + importer := svc.NewImporter(game) + result, err := importer.Import(context.Background(), archivePath, game, core.ImportOptions{}) + require.Error(t, err) + require.Nil(t, result) + require.Contains(t, err.Error(), "base pak") + require.Equal(t, 0, src.compileCalls) +} + +// TestImportMod_DeployCompile_CompileFailureLeavesNoPartialArtifact proves a +// mid-compile failure never lands a partial/uncompiled file in the cache +// (#173 - "never silently cache an uncompiled .exmodz"). +func TestImportMod_DeployCompile_CompileFailureLeavesNoPartialArtifact(t *testing.T) { + installDir := t.TempDir() + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + require.NoError(t, os.WriteFile(basePak, []byte("fake-base-pak"), 0o644)) + + cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} + svc, err := core.NewService(cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + src := &failingCompilerSource{fakeCompilerSource: &fakeCompilerSource{}} + svc.RegisterSource(src) + + game := &domain.Game{ + ID: "icarus", + InstallPath: installDir, + ModPath: t.TempDir(), + DeployMode: domain.DeployCompile, + SourceIDs: map[string]string{"fake-compiler": "external-icarus-id"}, + } + require.NoError(t, svc.AddGame(game)) + + tempDir := t.TempDir() + archivePath := filepath.Join(tempDir, "Bear_Mount.exmodz") + require.NoError(t, os.WriteFile(archivePath, []byte("fake-exmodz-bytes"), 0o644)) + + importer := svc.NewImporter(game) + result, err := importer.Import(context.Background(), archivePath, game, core.ImportOptions{}) + require.Error(t, err) + require.Nil(t, result) + require.Contains(t, err.Error(), "compiling mod") + + _, statErr := os.Stat(filepath.Join(cfg.CacheDir, game.ID)) + require.True(t, os.IsNotExist(statErr), "no partial cache entry should have been created") +} + +// TestImportMod_DeployCompile_StandaloneImporterFailsLoud proves an +// Importer constructed without Service context (core.NewImporter, used +// directly in older tests) still fails loud rather than silently caching an +// uncompiled .exmodz - it simply has no compiler resolver to consult. +func TestImportMod_DeployCompile_StandaloneImporterFailsLoud(t *testing.T) { + tempDir := t.TempDir() + cacheDir := filepath.Join(tempDir, "cache") + archivePath := filepath.Join(tempDir, "Bear_Mount.exmodz") + require.NoError(t, os.WriteFile(archivePath, []byte("fake-exmodz-bytes"), 0o644)) + + modCache := cache.New(cacheDir) + game := &domain.Game{ID: "icarus", DeployMode: domain.DeployCompile} + + importer := core.NewImporter(modCache) + result, err := importer.Import(context.Background(), archivePath, game, core.ImportOptions{}) + require.Error(t, err) + require.Nil(t, result) + require.Contains(t, err.Error(), "compiler") +} From ccd7e6819bb23beef6ab61b122c935bfbd6fd024 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 15:33:40 -0400 Subject: [PATCH 43/96] feat: colorize CLI output by default on a TTY (#112) colorEnabled() now checks stdout's terminal capability (via termenv, already a direct dep) in addition to the existing --no-color/NO_COLOR gate, so piped/redirected output stays plain without an opt-out and --json output is never colored. Extends the deploy/verify-only colorGreen/Red/Yellow accents to list, status, search, update, conflicts, and mod show: bolded table headers, whole-row tinting for list's enabled/disabled/undeployed states, last-column inline coloring for search's [installed] marker and update's POLICY column, accented status/error/warning lines, and the existing checkmark convention extended to update and mod's confirmations. Table color is applied only to already-tabwriter-flushed text (never to a cell before it reaches text/tabwriter, which pads columns by raw byte length and would misalign them) via a new printTable helper - verified empirically and guarded by alignment-preserving tests. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 4 +- README.md | 2 + cmd/lmm/color_test.go | 151 ++++++++++++++++++++++++++++++++ cmd/lmm/conflicts.go | 2 +- cmd/lmm/conflicts_color_test.go | 30 +++++++ cmd/lmm/list.go | 29 +++++- cmd/lmm/list_color_test.go | 115 ++++++++++++++++++++++++ cmd/lmm/mod.go | 20 +++-- cmd/lmm/mod_color_test.go | 44 ++++++++++ cmd/lmm/mod_show_color_test.go | 62 +++++++++++++ cmd/lmm/root.go | 84 +++++++++++++++++- cmd/lmm/root_color_test.go | 41 +++++++++ cmd/lmm/search.go | 14 ++- cmd/lmm/search_color_test.go | 109 +++++++++++++++++++++++ cmd/lmm/status.go | 12 ++- cmd/lmm/status_color_test.go | 64 ++++++++++++++ cmd/lmm/update.go | 32 +++++-- cmd/lmm/update_color_test.go | 78 +++++++++++++++++ 18 files changed, 866 insertions(+), 27 deletions(-) create mode 100644 cmd/lmm/color_test.go create mode 100644 cmd/lmm/conflicts_color_test.go create mode 100644 cmd/lmm/list_color_test.go create mode 100644 cmd/lmm/mod_color_test.go create mode 100644 cmd/lmm/mod_show_color_test.go create mode 100644 cmd/lmm/root_color_test.go create mode 100644 cmd/lmm/search_color_test.go create mode 100644 cmd/lmm/status_color_test.go create mode 100644 cmd/lmm/update_color_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 703923d..a271e84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [1.27.1] - 2026-07-30 +### Added + +- CLI output is now colorized by default when stdout is a terminal, extending the existing `colorGreen`/`colorRed`/`colorYellow` accent mechanism (previously only used by `deploy`/`verify`) across `list`, `status`, `search`, `update`, `conflicts`, and `mod show`: table headers are bolded; a disabled mod's row in `lmm list -v` is dimmed and an enabled-but-undeployed row is accented yellow; `search`'s `[installed]` marker and `update`'s POLICY column color per row; `conflicts`' stale winner suffix, `mod show`'s pinned policy and lock line, and the `Enabled`/`Disabled` counts in `lmm status -g ` are accented; and the existing `✓`/`✗` success/failure markers extend to `update` and `mod`'s confirmation lines. Detection is TTY-aware (piped/redirected output stays plain) and layers on top of the existing `--no-color` flag and `NO_COLOR` env var, which continue to work unchanged; `--json` output is never colored. Table color is applied only to already-tabwriter-padded text (bolded headers, whole-row tints, or a table's genuinely last column) — never to interior cell values before they reach `text/tabwriter`, which pads columns by raw byte length and would misalign them (#112) ### Fixed diff --git a/README.md b/README.md index 1d72743..d7ac933 100644 --- a/README.md +++ b/README.md @@ -859,6 +859,8 @@ A `directory` source now shows up with real capabilities in `lmm source list` (` | `--no-hooks` | | Disable all hooks at runtime | | `--no-color` | | Disable colored output (respects NO_COLOR env) | +Output is colorized by default whenever stdout is a terminal (headers, status accents like enabled/disabled/pinned, success/warning/error markers); piped or redirected output stays plain automatically, and `--json` output is never colored. Disable explicitly with `--no-color` or the `NO_COLOR` environment variable. + ### Commands | Command | Description | diff --git a/cmd/lmm/color_test.go b/cmd/lmm/color_test.go new file mode 100644 index 0000000..e71781c --- /dev/null +++ b/cmd/lmm/color_test.go @@ -0,0 +1,151 @@ +package main + +import ( + "bytes" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// withColorCapableStdout forces stdoutColorCapable() to report a +// color-capable TTY for the duration of the test, without needing a real +// pty - the seam tests override to simulate "running interactively". +func withColorCapableStdout(t *testing.T, capable bool) { + t.Helper() + old := stdoutColorCapable + stdoutColorCapable = func() bool { return capable } + t.Cleanup(func() { stdoutColorCapable = old }) +} + +func resetColorFlags(t *testing.T) { + t.Helper() + oldNoColor := noColor + noColor = false + t.Cleanup(func() { noColor = oldNoColor }) + + oldEnv, hadEnv := os.LookupEnv("NO_COLOR") + require.NoError(t, os.Unsetenv("NO_COLOR")) + t.Cleanup(func() { + if hadEnv { + require.NoError(t, os.Setenv("NO_COLOR", oldEnv)) + } else { + require.NoError(t, os.Unsetenv("NO_COLOR")) + } + }) +} + +func TestColorEnabled_DefaultsOffWhenStdoutIsNotATerminal(t *testing.T) { + resetColorFlags(t) + withColorCapableStdout(t, false) + + assert.False(t, colorEnabled(), "piped/redirected stdout must never emit color by default") +} + +func TestColorEnabled_OnByDefaultWhenStdoutIsATerminal(t *testing.T) { + resetColorFlags(t) + withColorCapableStdout(t, true) + + assert.True(t, colorEnabled(), "a color-capable TTY with no opt-out should color by default") +} + +func TestColorEnabled_NoColorFlagWinsOverTTY(t *testing.T) { + resetColorFlags(t) + withColorCapableStdout(t, true) + oldNoColor := noColor + noColor = true + t.Cleanup(func() { noColor = oldNoColor }) + + assert.False(t, colorEnabled(), "--no-color must disable color even on a real TTY") +} + +func TestColorEnabled_NoColorEnvWinsOverTTY(t *testing.T) { + resetColorFlags(t) + withColorCapableStdout(t, true) + require.NoError(t, os.Setenv("NO_COLOR", "1")) + + assert.False(t, colorEnabled(), "NO_COLOR must disable color even on a real TTY") +} + +func TestColorHelpers_NoOpWhenColorDisabled(t *testing.T) { + resetColorFlags(t) + withColorCapableStdout(t, false) + + assert.Equal(t, "text", colorGreen("text")) + assert.Equal(t, "text", colorRed("text")) + assert.Equal(t, "text", colorYellow("text")) + assert.Equal(t, "text", colorBold("text")) + assert.Equal(t, "text", colorDim("text")) +} + +func TestColorHelpers_WrapWhenColorEnabled(t *testing.T) { + resetColorFlags(t) + withColorCapableStdout(t, true) + + assert.Equal(t, ansiGreen+"text"+ansiReset, colorGreen("text")) + assert.Equal(t, ansiRed+"text"+ansiReset, colorRed("text")) + assert.Equal(t, ansiYellow+"text"+ansiReset, colorYellow("text")) + assert.Equal(t, ansiBold+"text"+ansiReset, colorBold("text")) + assert.Equal(t, ansiDim+"text"+ansiReset, colorDim("text")) +} + +// TestPrintTable_ColorNeverShiftsColumnAlignment guards the exact regression +// this feature risks: text/tabwriter computes column padding from raw byte +// length, so injecting ANSI escapes into a cell BEFORE it reaches the +// tabwriter would inflate that cell's measured width and misalign every +// column after it. printTable colors the ALREADY-flushed, plain-padded +// text instead - stripping ANSI from its output must reproduce the +// plain-mode text byte-for-byte, for any row-color choice. +func TestPrintTable_ColorNeverShiftsColumnAlignment(t *testing.T) { + resetColorFlags(t) + + build := func() *bytes.Buffer { + var buf bytes.Buffer + buf.WriteString("ID\tNAME\tENABLED\n") + buf.WriteString("--\t----\t-------\n") + buf.WriteString("modA\tSome Mod\tyes\n") + buf.WriteString("modB-longer-id\tAnother\tno\n") + // tabwriter enforcement of column widths happens on Flush of a live + // writer; simulate its already-flushed output directly since the + // production code always calls printTable post-Flush. + return &buf + } + + plainBuf := build() + var plainOut bytes.Buffer + withColorCapableStdout(t, false) + require.NoError(t, printTableTo(&plainOut, plainBuf, 2, nil)) + + coloredBuf := build() + var coloredOut bytes.Buffer + withColorCapableStdout(t, true) + rowColor := func(i int) func(string) string { + if i == 1 { + return colorRed + } + return colorGreen + } + require.NoError(t, printTableTo(&coloredOut, coloredBuf, 2, rowColor)) + + stripped := stripANSI(coloredOut.String()) + assert.Equal(t, plainOut.String(), stripped, "color must not change the visible text or alignment") + assert.Contains(t, coloredOut.String(), ansiBold, "header line should be bolded when color is enabled") + assert.Contains(t, coloredOut.String(), ansiGreen) + assert.Contains(t, coloredOut.String(), ansiRed) +} + +func stripANSI(s string) string { + for { + start := strings.Index(s, "\x1b[") + if start == -1 { + return s + } + end := strings.Index(s[start:], "m") + if end == -1 { + return s + } + s = s[:start] + s[start+end+1:] + } +} diff --git a/cmd/lmm/conflicts.go b/cmd/lmm/conflicts.go index 72800f4..e91ebbe 100644 --- a/cmd/lmm/conflicts.go +++ b/cmd/lmm/conflicts.go @@ -148,7 +148,7 @@ func doConflicts(ctx context.Context, svc *core.Service, game *domain.Game) erro fmt.Println() winner := c.LoadOrderWinner.Name if c.Stale { - winner += " (stale — redeploy to apply)" + winner += " " + colorYellow("(stale — redeploy to apply)") } fmt.Printf(" Winner: %s\n", winner) fmt.Println() diff --git a/cmd/lmm/conflicts_color_test.go b/cmd/lmm/conflicts_color_test.go new file mode 100644 index 0000000..9439276 --- /dev/null +++ b/cmd/lmm/conflicts_color_test.go @@ -0,0 +1,30 @@ +package main + +import ( + "context" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestDoConflicts_Stale_ColorPath: the "(stale — redeploy to apply)" suffix +// is a pending/attention-worthy state (yellow, per the repo's palette), and +// is a plain non-tabular line so no tabwriter alignment concern applies. +func TestDoConflicts_Stale_ColorPath(t *testing.T) { + svc, game := setupConflictsTest(t) + seedTwinConflictFixture(t, svc, game) + require.NoError(t, svc.NewProfileManager().ReorderMods(game.ID, "default", []domain.ModReference{ + {SourceID: "src", ModID: "b", Version: "1.0"}, + {SourceID: "src", ModID: "a", Version: "1.0"}, + })) + + resetColorFlags(t) + withColorCapableStdout(t, true) + out := captureStdout(t, func() error { + return doConflicts(context.Background(), svc, game) + }) + + assert.Contains(t, out, "Winner: Mod A "+colorYellow("(stale — redeploy to apply)")) +} diff --git a/cmd/lmm/list.go b/cmd/lmm/list.go index 83e9528..3011d2f 100644 --- a/cmd/lmm/list.go +++ b/cmd/lmm/list.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "context" "encoding/json" "fmt" @@ -146,7 +147,8 @@ func doList(cmd *cobra.Command, service *core.Service, game *domain.Game) error } fmt.Println() - w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + var buf bytes.Buffer + w := tabwriter.NewWriter(&buf, 0, 0, 2, ' ', 0) header := "ID\tNAME\tVERSION\tAUTHOR" sep := "--\t----\t-------\t------" if verbose { @@ -195,6 +197,31 @@ func doList(cmd *cobra.Command, service *core.Service, game *domain.Game) error return fmt.Errorf("flushing output: %w", err) } + // Row tinting only makes sense next to the columns it explains: ENABLED + // and DEPLOYED are verbose-only, so an anomaly (disabled, or enabled but + // not yet deployed) would be an unexplained color in the non-verbose + // table. Enabled+deployed - the common, unremarkable case - stays + // untinted (accent, not christmas tree). + var rowColor func(int) func(string) string + if verbose { + rowColor = func(i int) func(string) string { + if i < 0 || i >= len(mods) { + return nil + } + switch { + case !mods[i].Enabled: + return colorDim + case !mods[i].Deployed: + return colorYellow + default: + return nil + } + } + } + if err := printTable(&buf, 2, rowColor); err != nil { + return fmt.Errorf("writing table: %w", err) + } + return nil } diff --git a/cmd/lmm/list_color_test.go b/cmd/lmm/list_color_test.go new file mode 100644 index 0000000..1ad4149 --- /dev/null +++ b/cmd/lmm/list_color_test.go @@ -0,0 +1,115 @@ +package main + +import ( + "strings" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/core" + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// seedModWithState installs modID/name with an explicit enabled/deployed +// combination (seedDeployableMod always seeds Enabled: true, Deployed: +// false, which isn't enough to exercise every row-tint branch). +func seedModWithState(t *testing.T, svc *core.Service, game *domain.Game, modID, name string, enabled, deployed bool) { + t.Helper() + + require.NoError(t, svc.SaveInstalledMod(&domain.InstalledMod{ + Mod: domain.Mod{ID: modID, SourceID: "src", Name: name, Version: "1.0", GameID: game.ID}, + ProfileName: "default", + UpdatePolicy: domain.UpdateNotify, + Enabled: enabled, + Deployed: deployed, + })) + pm := svc.NewProfileManager() + if _, err := pm.Get(game.ID, "default"); err != nil { + require.ErrorIs(t, err, domain.ErrProfileNotFound) + _, err := pm.Create(game.ID, "default") + require.NoError(t, err) + } + require.NoError(t, pm.AddMod(game.ID, "default", domain.ModReference{SourceID: "src", ModID: modID, Version: "1.0"})) +} + +func rowFor(out, name string) string { + for _, l := range strings.Split(out, "\n") { + if strings.Contains(l, name) { + return l + } + } + return "" +} + +// TestList_Verbose_PlainWhenColorDisabled is the byte-stability regression +// guard: with color off (the default for piped/non-TTY output, and every +// test that doesn't force stdoutColorCapable), `lmm list -v` output must +// carry no ANSI escapes at all, regardless of each mod's enabled/deployed +// state. +func TestList_Verbose_PlainWhenColorDisabled(t *testing.T) { + svc, game := setupDoDeployTest(t) + seedModWithState(t, svc, game, "a", "Enabled Deployed", true, true) + seedModWithState(t, svc, game, "b", "Disabled Mod", false, false) + seedModWithState(t, svc, game, "c", "Enabled Undeployed", true, false) + + out := listVerbose(t, svc, game, false) + + assert.NotContains(t, out, "\x1b[", "plain output must never contain ANSI escapes") + assert.Contains(t, out, "Disabled Mod") + assert.Contains(t, out, "Enabled Undeployed") +} + +func TestList_Verbose_RowTinting(t *testing.T) { + tests := []struct { + name string + enabled bool + deployed bool + wantANSI string + wantNoOtherFor []string + }{ + {name: "disabled mod row is dimmed", enabled: false, deployed: false, wantANSI: ansiDim}, + {name: "enabled but undeployed row is yellow", enabled: true, deployed: false, wantANSI: ansiYellow}, + {name: "enabled and deployed row is untinted", enabled: true, deployed: true, wantANSI: ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + svc, game := setupDoDeployTest(t) + // setupDoDeployTest forces noColor=true; undo it so the + // stdoutColorCapable override below actually takes effect. + resetColorFlags(t) + seedModWithState(t, svc, game, "x", "Target Mod", tt.enabled, tt.deployed) + + withColorCapableStdout(t, true) + out := listVerbose(t, svc, game, false) + + row := rowFor(out, "Target Mod") + require.NotEmpty(t, row) + if tt.wantANSI == "" { + assert.NotContains(t, row, "\x1b[", "an enabled+deployed row should not be tinted") + } else { + assert.Contains(t, row, tt.wantANSI) + } + }) + } +} + +// TestList_Verbose_ColorNeverBreaksAlignment: stripping ANSI from a +// color-enabled run must reproduce the color-disabled run byte-for-byte - +// text/tabwriter pads by raw byte length, so this is the guard against a +// silent column-alignment regression (see printTable's doc comment). +func TestList_Verbose_ColorNeverBreaksAlignment(t *testing.T) { + svc, game := setupDoDeployTest(t) + resetColorFlags(t) + seedModWithState(t, svc, game, "a", "Enabled Deployed", true, true) + seedModWithState(t, svc, game, "b", "Disabled Mod", false, false) + seedModWithState(t, svc, game, "c", "Enabled Undeployed", true, false) + + withColorCapableStdout(t, false) + plain := listVerbose(t, svc, game, false) + + withColorCapableStdout(t, true) + colored := listVerbose(t, svc, game, false) + + assert.Equal(t, plain, stripANSI(colored)) +} diff --git a/cmd/lmm/mod.go b/cmd/lmm/mod.go index 8e50a4e..5fc858c 100644 --- a/cmd/lmm/mod.go +++ b/cmd/lmm/mod.go @@ -230,7 +230,7 @@ func doModSetUpdate(service *core.Service, game *domain.Game, modID string) erro return fmt.Errorf("failed to update policy: %w", err) } - fmt.Printf("✓ %s update policy: %s", mod.Name, policyStr) + fmt.Printf("%s %s update policy: %s", colorGreen("✓"), mod.Name, policyStr) if modSetPin { fmt.Printf(" (v%s)", mod.Version) } @@ -327,7 +327,7 @@ func doModLock(ctx context.Context, service *core.Service, game *domain.Game, mo return err } - fmt.Printf("✓ %s locked at v%s\n", mod.Name, target) + fmt.Printf("%s %s locked at v%s\n", colorGreen("✓"), mod.Name, target) // Locking is a metadata write, not a deploy (design decision): when the // target differs from what is actually installed, the game directory // won't match the lock until convergence, so say so. @@ -370,7 +370,7 @@ func doModUnlock(service *core.Service, game *domain.Game, modID string) error { return err } - fmt.Printf("✓ %s unlocked (update policy: %s)\n", mod.Name, policyToString(mod.UpdatePolicy)) + fmt.Printf("%s %s unlocked (update policy: %s)\n", colorGreen("✓"), mod.Name, policyToString(mod.UpdatePolicy)) return nil } @@ -422,7 +422,7 @@ func doModEnable(ctx context.Context, service *core.Service, game *domain.Game, return nil } - fmt.Printf("✓ Enabled: %s\n", mod.Name) + fmt.Printf("%s Enabled: %s\n", colorGreen("✓"), mod.Name) return nil } @@ -473,7 +473,7 @@ func doModDisable(ctx context.Context, service *core.Service, game *domain.Game, return nil } - fmt.Printf("✓ Disabled: %s (files removed from game, kept in cache)\n", mod.Name) + fmt.Printf("%s Disabled: %s (files removed from game, kept in cache)\n", colorGreen("✓"), mod.Name) return nil } @@ -631,7 +631,7 @@ func doModShow(ctx context.Context, svc *core.Service, game *domain.Game, modID // Human-readable output fmt.Printf("%s\n", strings.Repeat("=", 60)) - fmt.Printf("%s\n", mod.Name) + fmt.Printf("%s\n", colorBold(mod.Name)) fmt.Printf("%s\n", strings.Repeat("=", 60)) fmt.Printf("ID: %s Version: %s Author: %s\n", mod.ID, mod.Version, mod.Author) if mod.Category != "" { @@ -668,7 +668,11 @@ func doModShow(ctx context.Context, svc *core.Service, game *domain.Game, modID if installedInfo != nil { fmt.Println() fmt.Printf("Installed: v%s (profile: %s)\n", installedInfo.Version, installedInfo.Profile) - fmt.Printf(" Update policy: %s\n", installedInfo.UpdatePolicy) + policyDisplay := installedInfo.UpdatePolicy + if policyDisplay == "pinned" { + policyDisplay = colorYellow(policyDisplay) + } + fmt.Printf(" Update policy: %s\n", policyDisplay) if installedInfo.Locked { lockLine := "locked at v" + installedInfo.LockedVersion // Locking is a metadata write, not a deploy (same #97 design @@ -678,7 +682,7 @@ func doModShow(ctx context.Context, svc *core.Service, game *domain.Game, modID if installedInfo.LockedVersion != installedInfo.Version { lockLine += " — run 'lmm profile apply' to converge" } - fmt.Printf(" Lock: %s\n", lockLine) + fmt.Printf(" Lock: %s\n", colorYellow(lockLine)) } else { fmt.Println(" Lock: none") } diff --git a/cmd/lmm/mod_color_test.go b/cmd/lmm/mod_color_test.go new file mode 100644 index 0000000..203cb82 --- /dev/null +++ b/cmd/lmm/mod_color_test.go @@ -0,0 +1,44 @@ +package main + +import ( + "context" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/stretchr/testify/assert" +) + +// TestDoModSetUpdate_SuccessCheckmark_ColorPath and +// TestDoModLock_SuccessCheckmark_ColorPath extend mod.go's existing "✓ ..." +// success lines with the same colorGreen("✓") convention deploy.go/verify.go +// already use - see update.go's identical extension for the update command. +func TestDoModSetUpdate_SuccessCheckmark_ColorPath(t *testing.T) { + svc, game, _ := setupDoModLockTest(t) + seedLockableMod(t, svc, game, "a", "Mod A", "1.5") + modSetAuto = true + t.Cleanup(func() { modSetAuto = false }) + + resetColorFlags(t) + withColorCapableStdout(t, true) + out := captureStdout(t, func() error { + return doModSetUpdate(svc, game, "a") + }) + + assert.Contains(t, out, colorGreen("✓")+" Mod A update policy: auto") +} + +func TestDoModLock_SuccessCheckmark_ColorPath(t *testing.T) { + svc, game, src := setupDoModLockTest(t) + seedLockableMod(t, svc, game, "a", "Mod A", "1.0") + src.AddMod(&domain.Mod{ID: "a", SourceID: "src", GameID: game.ID}, []domain.DownloadableFile{ + {ID: "f1", Version: "1.0", Category: "MAIN"}, + }) + + resetColorFlags(t) + withColorCapableStdout(t, true) + out := captureStdout(t, func() error { + return doModLock(context.Background(), svc, game, "a", "") + }) + + assert.Contains(t, out, colorGreen("✓")+" Mod A locked at v1.0") +} diff --git a/cmd/lmm/mod_show_color_test.go b/cmd/lmm/mod_show_color_test.go new file mode 100644 index 0000000..dd5e7b2 --- /dev/null +++ b/cmd/lmm/mod_show_color_test.go @@ -0,0 +1,62 @@ +package main + +import ( + "context" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestDoModShow_ColorPath_PlainByDefault is the byte-stability guard: with +// color off (the default), mod show output must carry no ANSI escapes, +// whether or not the mod is locked. +func TestDoModShow_ColorPath_PlainByDefault(t *testing.T) { + svc, game, src := setupDoModLockTest(t) + seedLockableMod(t, svc, game, "a", "Mod A", "1.5") + src.AddMod(&domain.Mod{ID: "a", SourceID: "src", GameID: game.ID, Name: "Mod A", Version: "1.5"}, nil) + require.NoError(t, svc.NewProfileManager().SetModLock(game.ID, "default", "src", "a", "1.2.3")) + + out := captureStdout(t, func() error { + return doModShow(context.Background(), svc, game, "a") + }) + + assert.NotContains(t, out, "\x1b[") +} + +// TestDoModShow_ColorPath_NameBolded_LockAccented: the mod's name banner is +// bolded, and a lock (a held-back/pending state) is accented yellow - +// matching the repo's established "pending"=yellow mapping. +func TestDoModShow_ColorPath_NameBolded_LockAccented(t *testing.T) { + svc, game, src := setupDoModLockTest(t) + seedLockableMod(t, svc, game, "a", "Mod A", "1.5") + src.AddMod(&domain.Mod{ID: "a", SourceID: "src", GameID: game.ID, Name: "Mod A", Version: "1.5"}, nil) + require.NoError(t, svc.NewProfileManager().SetModLock(game.ID, "default", "src", "a", "1.2.3")) + + resetColorFlags(t) + withColorCapableStdout(t, true) + out := captureStdout(t, func() error { + return doModShow(context.Background(), svc, game, "a") + }) + + assert.Contains(t, out, colorBold("Mod A")) + assert.Contains(t, out, colorYellow("locked at v1.2.3 — run 'lmm profile apply' to converge")) +} + +// TestDoModShow_ColorPath_PinnedPolicyAccented guards the "pinned" update +// policy - the issue's own example of a yellow/pending accent. +func TestDoModShow_ColorPath_PinnedPolicyAccented(t *testing.T) { + svc, game, src := setupDoModLockTest(t) + seedLockableMod(t, svc, game, "a", "Mod A", "1.5") + src.AddMod(&domain.Mod{ID: "a", SourceID: "src", GameID: game.ID, Name: "Mod A", Version: "1.5"}, nil) + require.NoError(t, svc.SetModUpdatePolicy("src", "a", game.ID, "default", domain.UpdatePinned)) + + resetColorFlags(t) + withColorCapableStdout(t, true) + out := captureStdout(t, func() error { + return doModShow(context.Background(), svc, game, "a") + }) + + assert.Contains(t, out, "Update policy: "+colorYellow("pinned")) +} diff --git a/cmd/lmm/root.go b/cmd/lmm/root.go index 16c60a5..5a90563 100644 --- a/cmd/lmm/root.go +++ b/cmd/lmm/root.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "context" "errors" "fmt" @@ -8,6 +9,7 @@ import ( "os" "os/signal" "path/filepath" + "strings" "syscall" "github.com/DonovanMods/linux-mod-manager/internal/core" @@ -18,6 +20,7 @@ import ( "github.com/DonovanMods/linux-mod-manager/internal/source/nexusmods" "github.com/DonovanMods/linux-mod-manager/internal/storage/config" + "github.com/muesli/termenv" "github.com/spf13/cobra" ) @@ -90,8 +93,21 @@ func init() { rootCmd.PersistentFlags().BoolVar(&noColor, "no-color", false, "disable colored output (NO_COLOR env is also honored)") } -// colorEnabled returns true if colored output should be used (respects --no-color and NO_COLOR env). -// NO_COLOR: if set (any value), color is disabled per https://no-color.org +// stdoutColorCapable reports whether the live os.Stdout is a color-capable +// terminal (not a pipe, redirect, or non-interactive runner). A function +// var, not a direct termenv.ColorProfile() call, so it re-resolves the +// CURRENT os.Stdout on every call - tests that swap os.Stdout for an +// os.Pipe (see captureStdout) get a truthful "not a terminal" answer for +// free, and tests that need to simulate an interactive TTY can override +// this var directly instead of faking a pty. +var stdoutColorCapable = func() bool { + return termenv.NewOutput(os.Stdout).ColorProfile() != termenv.Ascii +} + +// colorEnabled returns true if colored output should be used: respects +// --no-color and NO_COLOR env (https://no-color.org) first, then falls back +// to TTY detection so piped/redirected output stays plain without an +// explicit opt-out. func colorEnabled() bool { if noColor { return false @@ -99,7 +115,7 @@ func colorEnabled() bool { if os.Getenv("NO_COLOR") != "" { return false } - return true + return stdoutColorCapable() } const ( @@ -107,6 +123,8 @@ const ( ansiGreen = "\033[32m" ansiRed = "\033[31m" ansiYellow = "\033[33m" + ansiBold = "\033[1m" + ansiDim = "\033[2m" ) // colorGreen returns s with green ANSI when color is enabled, otherwise s. @@ -133,6 +151,64 @@ func colorYellow(s string) string { return ansiYellow + s + ansiReset } +// colorBold returns s with bold ANSI when color is enabled, otherwise s. +func colorBold(s string) string { + if !colorEnabled() { + return s + } + return ansiBold + s + ansiReset +} + +// colorDim returns s with faint/dim ANSI when color is enabled, otherwise s. +// Used for negative-but-routine states (e.g. a disabled mod row) where a +// loud red would overstate the severity - accent, not alarm. +func colorDim(s string) string { + if !colorEnabled() { + return s + } + return ansiDim + s + ansiReset +} + +// printTable writes a fully-flushed text/tabwriter table (buf) to os.Stdout, +// bolding the header line and applying rowColor's per-row wrapper (nil for +// no tint) when color is enabled. headerLines is the number of leading +// lines to skip when indexing data rows (2: header + dashed separator). +// +// Color is applied ONLY to buf's already-rendered, already-padded text - +// never to a cell before it reaches the tabwriter. text/tabwriter computes +// column padding from raw byte length, so an ANSI-wrapped cell fed into it +// would inflate that cell's measured width and misalign every column after +// it (verified empirically). Wrapping an already-flushed line's start/end +// is safe: those bytes are invisible to the terminal and never shift where +// the real characters land. Do not colorize interior cell values before +// Fprintf-ing them into a tabwriter.Writer - use whole-row tinting (via +// rowColor) or, for a table's genuinely last column (nothing pads after it +// per tabwriter's own behavior), inline coloring of that one column instead. +func printTable(buf *bytes.Buffer, headerLines int, rowColor func(dataRowIndex int) func(string) string) error { + return printTableTo(os.Stdout, buf, headerLines, rowColor) +} + +// printTableTo is printTable's testable seam: same contract, explicit writer. +func printTableTo(out io.Writer, buf *bytes.Buffer, headerLines int, rowColor func(dataRowIndex int) func(string) string) error { + text := strings.TrimSuffix(buf.String(), "\n") + if text == "" { + return nil + } + lines := strings.Split(text, "\n") + if colorEnabled() { + lines[0] = colorBold(lines[0]) + if rowColor != nil { + for i := headerLines; i < len(lines); i++ { + if fn := rowColor(i - headerLines); fn != nil { + lines[i] = fn(lines[i]) + } + } + } + } + _, err := fmt.Fprintln(out, strings.Join(lines, "\n")) + return err +} + // Execute runs the root command. Exit codes: 0 = success, 1 = error, 2 = user cancelled. // When --json is set and an error occurs, prints {"error":"..."} to stdout before exiting. // Cancellation (ErrCancelled or context.Canceled) exits with code 2 without printing JSON, @@ -160,7 +236,7 @@ func reportError(err error) { if jsonOutput { fmt.Printf(`{"error":%q}`+"\n", err.Error()) } else { - fmt.Fprintf(os.Stderr, "Error: %v\n", err) + fmt.Fprintf(os.Stderr, "%s %v\n", colorRed("Error:"), err) } } diff --git a/cmd/lmm/root_color_test.go b/cmd/lmm/root_color_test.go new file mode 100644 index 0000000..bc4b1a1 --- /dev/null +++ b/cmd/lmm/root_color_test.go @@ -0,0 +1,41 @@ +package main + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestReportError_PlainWhenColorDisabled is the byte-stability guard: the +// existing "Error: %v" format must be unchanged when color is off (the +// default). +func TestReportError_PlainWhenColorDisabled(t *testing.T) { + oldJSON := jsonOutput + jsonOutput = false + t.Cleanup(func() { jsonOutput = oldJSON }) + + out, _ := captureStderrErr(t, func() error { + reportError(errors.New("boom")) + return nil + }) + + assert.Equal(t, "Error: boom\n", out) +} + +// TestReportError_ColorPath extends reportError's "Error:" prefix with the +// existing colorRed convention (deploy.go/verify.go's error markers). +func TestReportError_ColorPath(t *testing.T) { + oldJSON := jsonOutput + jsonOutput = false + t.Cleanup(func() { jsonOutput = oldJSON }) + + resetColorFlags(t) + withColorCapableStdout(t, true) + out, _ := captureStderrErr(t, func() error { + reportError(errors.New("boom")) + return nil + }) + + assert.Equal(t, colorRed("Error:")+" boom\n", out) +} diff --git a/cmd/lmm/search.go b/cmd/lmm/search.go index b8c7e20..595059b 100644 --- a/cmd/lmm/search.go +++ b/cmd/lmm/search.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "context" "encoding/json" "errors" @@ -280,7 +281,8 @@ func doSearch(ctx context.Context, service *core.Service, game *domain.Game, arg } // Print results - w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + var buf bytes.Buffer + w := tabwriter.NewWriter(&buf, 0, 0, 2, ' ', 0) if _, err := fmt.Fprintln(w, "ID\tNAME\tAUTHOR\tVERSION\tSOURCE\t"); err != nil { return fmt.Errorf("writing header: %w", err) } @@ -291,7 +293,12 @@ func doSearch(ctx context.Context, service *core.Service, game *domain.Game, arg for _, mod := range mods { installedMark := "" if installedKeys[domain.ModKey(mod.SourceID, mod.ID)] { - installedMark = "[installed]" + // Safe to color inline here specifically because it's the LAST + // column: text/tabwriter never pads after the final cell, so + // this cell's byte length (inflated by ANSI codes) can't + // corrupt any other column's alignment. Do not do this for an + // interior column - see printTable's doc comment. + installedMark = colorGreen("[installed]") } if _, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", mod.ID, @@ -307,6 +314,9 @@ func doSearch(ctx context.Context, service *core.Service, game *domain.Game, arg if err := w.Flush(); err != nil { return fmt.Errorf("flushing output: %w", err) } + if err := printTable(&buf, 2, nil); err != nil { + return fmt.Errorf("writing table: %w", err) + } if verbose { fmt.Printf("\nShowing %d of %d results.\n", len(mods), totalResults) diff --git a/cmd/lmm/search_color_test.go b/cmd/lmm/search_color_test.go new file mode 100644 index 0000000..5b4fd01 --- /dev/null +++ b/cmd/lmm/search_color_test.go @@ -0,0 +1,109 @@ +package main + +import ( + "context" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/core" + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/source" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// twoModSource is a minimal ModSource returning a fixed two-mod result set, +// so search's results table has both an installed and a not-installed row. +type twoModSource struct{ id string } + +func (s *twoModSource) ID() string { return s.id } +func (s *twoModSource) Name() string { return s.id } +func (s *twoModSource) AuthURL() string { return "" } +func (s *twoModSource) ExchangeToken(context.Context, string) (*source.Token, error) { + return nil, nil +} +func (s *twoModSource) Search(context.Context, source.SearchQuery) (source.SearchResult, error) { + return source.SearchResult{ + Mods: []domain.Mod{ + {ID: "m1", SourceID: s.id, Name: "Installed Mod"}, + {ID: "m2", SourceID: s.id, Name: "Not Installed Mod"}, + }, + TotalCount: 2, + }, nil +} +func (s *twoModSource) GetMod(context.Context, string, string) (*domain.Mod, error) { return nil, nil } +func (s *twoModSource) GetDependencies(context.Context, *domain.Mod) ([]domain.ModReference, error) { + return nil, nil +} +func (s *twoModSource) GetModFiles(context.Context, *domain.Mod) ([]domain.DownloadableFile, error) { + return nil, nil +} +func (s *twoModSource) GetDownloadURL(context.Context, *domain.Mod, string) (string, error) { + return "", nil +} +func (s *twoModSource) CheckUpdates(context.Context, []domain.InstalledMod) ([]domain.Update, error) { + return nil, nil +} + +// setupSearchColorTest wires a real core.Service around a twoModSource, with +// "m1" already installed - so doSearch's real "already installed" and table +// rendering code paths run end to end. +func setupSearchColorTest(t *testing.T) (*core.Service, *domain.Game) { + t.Helper() + spy := &twoModSource{id: "src"} + svc, err := core.NewService(core.ServiceConfig{ + ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir(), + }) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + svc.RegisterSource(spy) + + game := &domain.Game{ + ID: "g1", Name: "Game", ModPath: t.TempDir(), + SourceIDs: map[string]string{spy.id: ""}, + } + require.NoError(t, svc.AddGame(game)) + + require.NoError(t, svc.SaveInstalledMod(&domain.InstalledMod{ + Mod: domain.Mod{ID: "m1", SourceID: spy.id, Name: "Installed Mod", Version: "1.0", GameID: game.ID}, + ProfileName: "default", + UpdatePolicy: domain.UpdateNotify, + Enabled: true, + })) + pm := svc.NewProfileManager() + _, err = pm.Create(game.ID, "default") + require.NoError(t, err) + require.NoError(t, pm.AddMod(game.ID, "default", domain.ModReference{SourceID: spy.id, ModID: "m1", Version: "1.0"})) + + withSearchFlags(t, spy.id, 10) + return svc, game +} + +func TestDoSearch_InstalledMarker_PlainWhenColorDisabled(t *testing.T) { + svc, game := setupSearchColorTest(t) + + out := captureStdout(t, func() error { + return doSearch(context.Background(), svc, game, []string{"query"}) + }) + + assert.NotContains(t, out, "\x1b[") + assert.Contains(t, out, "[installed]") +} + +func TestDoSearch_InstalledMarker_GreenWhenTTY_AlignmentUnaffected(t *testing.T) { + svc, game := setupSearchColorTest(t) + resetColorFlags(t) + + withColorCapableStdout(t, false) + plain := captureStdout(t, func() error { + return doSearch(context.Background(), svc, game, []string{"query"}) + }) + + withColorCapableStdout(t, true) + colored := captureStdout(t, func() error { + return doSearch(context.Background(), svc, game, []string{"query"}) + }) + + assert.Contains(t, colored, ansiGreen+"[installed]"+ansiReset) + assert.Contains(t, colored, ansiBold, "header line should be bolded") + assert.Equal(t, plain, stripANSI(colored), "color must not change the visible text or alignment") +} diff --git a/cmd/lmm/status.go b/cmd/lmm/status.go index 2c41afa..5eb7d7f 100644 --- a/cmd/lmm/status.go +++ b/cmd/lmm/status.go @@ -1,10 +1,12 @@ package main import ( + "bytes" "context" "encoding/json" "fmt" "os" + "strconv" "text/tabwriter" "time" @@ -72,7 +74,8 @@ func doStatus(service *core.Service) error { fmt.Println("Configured Games:") fmt.Println() - w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + var buf bytes.Buffer + w := tabwriter.NewWriter(&buf, 0, 0, 2, ' ', 0) if verbose { if _, err := fmt.Fprintln(w, "GAME\tID\tPATH\tLINK\tPROFILES\tMODS†"); err != nil { @@ -140,6 +143,9 @@ func doStatus(service *core.Service) error { if err := w.Flush(); err != nil { return fmt.Errorf("flushing output: %w", err) } + if err := printTable(&buf, 2, nil); err != nil { + return fmt.Errorf("writing table: %w", err) + } fmt.Println() if verbose { @@ -375,7 +381,9 @@ func showGameStatus(service *core.Service, gameID string) error { } } if len(mods) > 0 { - fmt.Printf(" Enabled: %d, Disabled: %d\n", enabled, disabled) + // Disabled is a routine, expected state (not an error), so it's + // dimmed rather than red - accent, not alarm. + fmt.Printf(" Enabled: %s, Disabled: %s\n", colorGreen(strconv.Itoa(enabled)), colorDim(strconv.Itoa(disabled))) } lastDeploy, err := service.GetLastDeployTime(gameID, defaultProfile.Name) diff --git a/cmd/lmm/status_color_test.go b/cmd/lmm/status_color_test.go new file mode 100644 index 0000000..f53a3a9 --- /dev/null +++ b/cmd/lmm/status_color_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestShowGameStatus_EnabledDisabledCounts_PlainWhenColorDisabled is the +// byte-stability guard: with color off (the default), the Enabled/Disabled +// summary line must carry no ANSI escapes. +func TestShowGameStatus_EnabledDisabledCounts_PlainWhenColorDisabled(t *testing.T) { + svc, game := setupDoDeployTest(t) + require.NoError(t, svc.AddGame(game)) + seedDeployableMod(t, svc, game, "1", "Enabled Mod", "a.esp") + seedModWithState(t, svc, game, "2", "Disabled Mod", false, false) + + out := captureStdout(t, func() error { + return showGameStatus(svc, game.ID) + }) + + assert.NotContains(t, out, "\x1b[") + assert.Contains(t, out, "Enabled: 1, Disabled: 1") +} + +func TestShowGameStatus_EnabledDisabledCounts_ColoredWhenTTY(t *testing.T) { + svc, game := setupDoDeployTest(t) + resetColorFlags(t) + require.NoError(t, svc.AddGame(game)) + seedDeployableMod(t, svc, game, "1", "Enabled Mod", "a.esp") + seedModWithState(t, svc, game, "2", "Disabled Mod", false, false) + + withColorCapableStdout(t, true) + out := captureStdout(t, func() error { + return showGameStatus(svc, game.ID) + }) + + assert.Contains(t, out, ansiGreen+"1"+ansiReset, "enabled count should be accented green") + assert.Contains(t, out, ansiDim+"1"+ansiReset, "disabled count should be a dim accent, not a loud red") +} + +// TestDoStatus_TableHeader_BoldedWhenTTY_AlignmentUnaffected guards the +// "Configured Games:" summary table: header bolding must not perturb the +// tabwriter-computed column alignment (see printTable's doc comment). +func TestDoStatus_TableHeader_BoldedWhenTTY_AlignmentUnaffected(t *testing.T) { + svc, game := setupDoDeployTest(t) + require.NoError(t, svc.AddGame(game)) + seedDeployableMod(t, svc, game, "1", "Test Mod", "a.esp") + + resetColorFlags(t) + withColorCapableStdout(t, false) + plain := captureStdout(t, func() error { + return doStatus(svc) + }) + + withColorCapableStdout(t, true) + colored := captureStdout(t, func() error { + return doStatus(svc) + }) + + assert.Contains(t, colored, ansiBold, "table header should be bolded when color is enabled") + assert.Equal(t, plain, stripANSI(colored), "color must not change the visible text or alignment") +} diff --git a/cmd/lmm/update.go b/cmd/lmm/update.go index 5afa83a..756c1c4 100644 --- a/cmd/lmm/update.go +++ b/cmd/lmm/update.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "context" "encoding/json" "errors" @@ -398,7 +399,8 @@ func doUpdate(ctx context.Context, service *core.Service, game *domain.Game, arg } // Display available updates with policy - w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + var buf bytes.Buffer + w := tabwriter.NewWriter(&buf, 0, 0, 2, ' ', 0) if _, err := fmt.Fprintf(w, "MOD\tCURRENT\tAVAILABLE\tPOLICY\n"); err != nil { return fmt.Errorf("writing header: %w", err) } @@ -428,6 +430,17 @@ func doUpdate(ctx context.Context, service *core.Service, game *domain.Game, arg autoUpdates = append(autoUpdates, update) } } + // Safe to color inline here specifically because POLICY is the + // LAST column - text/tabwriter never pads after the final cell, so + // this cell's inflated byte length can't misalign any column after + // it (see printTable's doc comment; do not do this for an interior + // column). + switch { + case isLocked: + policyStr = colorYellow(policyStr) + case strings.HasSuffix(policyStr, " ✓"): + policyStr = colorGreen(policyStr) + } if _, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", truncate(update.InstalledMod.Name, 40), update.InstalledMod.Version, @@ -440,8 +453,11 @@ func doUpdate(ctx context.Context, service *core.Service, game *domain.Game, arg if err := w.Flush(); err != nil { return fmt.Errorf("flushing output: %w", err) } + if err := printTable(&buf, 2, nil); err != nil { + return fmt.Errorf("writing table: %w", err) + } - fmt.Printf("\n%d update(s) available.\n", len(updates)) + fmt.Printf("\n%s\n", colorYellow(fmt.Sprintf("%d update(s) available.", len(updates)))) if skips := core.CountUpdateSkips(installed); skips.Total() > 0 { fmt.Println() printSkipped(skips) @@ -486,9 +502,9 @@ func doUpdate(ctx context.Context, service *core.Service, game *domain.Game, arg fmt.Printf("\nApplying %d auto-update(s)...\n", len(autoUpdates)) for _, update := range autoUpdates { if err := applyUpdate(ctx, service, game, update, profileName); err != nil { - fmt.Printf(" ✗ %s: %v\n", update.InstalledMod.Name, err) + fmt.Printf(" %s %s: %v\n", colorRed("✗"), update.InstalledMod.Name, err) } else { - fmt.Printf(" ✓ %s %s → %s\n", update.InstalledMod.Name, update.InstalledMod.Version, update.NewVersion) + fmt.Printf(" %s %s %s → %s\n", colorGreen("✓"), update.InstalledMod.Name, update.InstalledMod.Version, update.NewVersion) } } } @@ -512,9 +528,9 @@ func doUpdate(ctx context.Context, service *core.Service, game *domain.Game, arg fmt.Printf("\nApplying %d remaining update(s)...\n", len(notifyUpdates)) for _, update := range notifyUpdates { if err := applyUpdate(ctx, service, game, update, profileName); err != nil { - fmt.Printf(" ✗ %s: %v\n", update.InstalledMod.Name, err) + fmt.Printf(" %s %s: %v\n", colorRed("✗"), update.InstalledMod.Name, err) } else { - fmt.Printf(" ✓ %s %s → %s\n", update.InstalledMod.Name, update.InstalledMod.Version, update.NewVersion) + fmt.Printf(" %s %s %s → %s\n", colorGreen("✓"), update.InstalledMod.Name, update.InstalledMod.Version, update.NewVersion) } } } @@ -655,7 +671,7 @@ func applySingleUpdate(ctx context.Context, service *core.Service, game *domain. }) } - fmt.Printf("\n✓ Updated: %s %s → %s\n", mod.Name, oldVersion, newVersion) + fmt.Printf("\n%s Updated: %s %s → %s\n", colorGreen("✓"), mod.Name, oldVersion, newVersion) fmt.Println(" Previous version preserved for rollback") return nil } @@ -803,7 +819,7 @@ func doUpdateRollback(ctx context.Context, service *core.Service, game *domain.G }) } - fmt.Printf("\n✓ Rolled back: %s %s → %s\n", result.ModName, result.FromVersion, result.ToVersion) + fmt.Printf("\n%s Rolled back: %s %s → %s\n", colorGreen("✓"), result.ModName, result.FromVersion, result.ToVersion) return nil } diff --git a/cmd/lmm/update_color_test.go b/cmd/lmm/update_color_test.go new file mode 100644 index 0000000..7ef766c --- /dev/null +++ b/cmd/lmm/update_color_test.go @@ -0,0 +1,78 @@ +package main + +import ( + "context" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestApplySingleUpdate_SuccessCheckmark_ColorPath extends update.go's +// existing "✓ Updated: ..." success line with the same colorGreen("✓") +// convention deploy.go/verify.go already use. +func TestApplySingleUpdate_SuccessCheckmark_ColorPath(t *testing.T) { + svc, game, src := setupDoUpdateTest(t) + resetColorFlags(t) + mod := seedInstalledForUpdate(t, svc, game, "test-src", "mod1", "Mod One", "1.0", []string{"old-1"}, map[string][]byte{"mod1-old.esp": []byte("old-content")}) + src.AddMod(&domain.Mod{ID: "mod1", SourceID: "test-src", Name: "Mod One", Version: "2.0", GameID: "g1"}, + []domain.DownloadableFile{{ID: "new-1", FileName: "mod1-new.esp", IsPrimary: true}}) + src.AddDownload("new-1", []byte("new-content")) + + withColorCapableStdout(t, true) + out := captureStdout(t, func() error { + return applySingleUpdate(context.Background(), svc, game, mod, "default") + }) + + assert.Contains(t, out, colorGreen("✓")+" Updated: Mod One 1.0 → 2.0") +} + +// TestDoUpdateRollback_SuccessCheckmark_ColorPath mirrors the above for the +// rollback footer. +func TestDoUpdateRollback_SuccessCheckmark_ColorPath(t *testing.T) { + svc, game, src := setupDoUpdateTest(t) + mod := seedInstalledForUpdate(t, svc, game, "test-src", "mod1", "Mod One", "1.0", []string{"old-1"}, map[string][]byte{"mod1-old.esp": []byte("old-content")}) + src.AddMod(&domain.Mod{ID: "mod1", SourceID: "test-src", Name: "Mod One", Version: "2.0", GameID: "g1"}, + []domain.DownloadableFile{{ID: "new-1", FileName: "mod1-new.esp", IsPrimary: true}}) + src.AddDownload("new-1", []byte("new-content")) + require.NoError(t, captureStdoutOnlyErr(t, func() error { + return applySingleUpdate(context.Background(), svc, game, mod, "default") + })) + + resetColorFlags(t) + withColorCapableStdout(t, true) + out := captureStdout(t, func() error { + return doUpdateRollback(context.Background(), svc, game, "mod1") + }) + + assert.Contains(t, out, colorGreen("✓")+" Rolled back: Mod One 2.0 → 1.0") +} + +// TestDoUpdate_Table_ColorPath guards the update-available table: header +// bolded, the last column (POLICY, never padded by tabwriter - see +// printTable's doc comment) tinted per row, and the summary line accented - +// all without perturbing the plain-mode column alignment. +func TestDoUpdate_Table_ColorPath(t *testing.T) { + svc, game, src := setupDoUpdateTest(t) + game.SourceIDs = map[string]string{"test-src": "g1"} + seedInstalledForUpdate(t, svc, game, "test-src", "mod1", "Mod One", "1.0", nil, map[string][]byte{"mod1.esp": []byte("data")}) + src.AddMod(&domain.Mod{ID: "mod1", SourceID: "test-src", Name: "Mod One", Version: "2.0", GameID: "g1"}, nil) + + resetColorFlags(t) + withColorCapableStdout(t, false) + plain := captureStdout(t, func() error { + return doUpdate(context.Background(), svc, game, nil) + }) + assert.NotContains(t, plain, "\x1b[") + assert.Contains(t, plain, "1 update(s) available.") + + withColorCapableStdout(t, true) + colored := captureStdout(t, func() error { + return doUpdate(context.Background(), svc, game, nil) + }) + + assert.Contains(t, colored, ansiBold, "table header should be bolded") + assert.Contains(t, colored, ansiYellow+"1 update(s) available."+ansiReset, "the available-updates summary should be accented") + assert.Equal(t, plain, stripANSI(colored), "color must not change the visible text or alignment") +} From 4a8bacb0223cc93fb60a291b904cc970d954fa94 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 15:36:23 -0400 Subject: [PATCH 44/96] fix: include compile in deploy_mode validation message (#172 review) internal/storage/config/games.go's deploy_mode rejection message still listed "(valid: extract, copy)", stale from before the develop merge added compile as a real DeployMode value - actively misleading for the exact games.yaml typo scenario #172 exists to fix. Introduced domain.ValidLinkMethods/ValidDeployModes as the single source of truth and switched all six error sites (config.go, games.go x2, profiles.go x2, cmd/lmm/game.go) to interpolate them instead of a hand-written list, closing off the same staleness class repo-wide rather than just the flagged instance. Added a RED-first assertion to TestLoadGames_RejectsUnknownDeployMode pinning "compile" in the message. Also: CHANGELOG's #172 entry now names lmm game detect's steam-games.yaml deploy_mode explicitly (review Minor #2); report-172.md updated to flag its now-stale "no other callers" claim and record this round's changes (review Minor #1). Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 2 +- cmd/lmm/game.go | 4 ++-- internal/domain/game.go | 9 +++++++++ internal/storage/config/config.go | 4 ++-- internal/storage/config/config_test.go | 4 ++++ internal/storage/config/games.go | 8 ++++---- internal/storage/config/profiles.go | 8 ++++---- 7 files changed, 26 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c20ebd2..371a99f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- An unrecognized, non-empty `link_method` (`games.yaml`, profile files, imported profiles) or `deploy_mode` (`games.yaml`) is now a load-time error naming the field, the offending value, the owning game/profile, and the valid options — instead of silently falling back to the default (`symlink`/`extract`). **Breaking for configs that were already silently misbehaving:** a typo like `deploy_mode: compil` previously ran as `extract` with no warning; it now refuses to load until fixed. An empty/absent value is unaffected and keeps today's default exactly (#172) +- An unrecognized, non-empty `link_method` (`games.yaml`, profile files, imported profiles) or `deploy_mode` (`games.yaml`; also `lmm game detect`'s `steam-games.yaml`) is now a load-time error naming the field, the offending value, the owning game/profile, and the valid options — instead of silently falling back to the default (`symlink`/`extract`). **Breaking for configs that were already silently misbehaving:** a typo like `deploy_mode: compil` previously ran as `extract` with no warning; it now refuses to load until fixed. An empty/absent value is unaffected and keeps today's default exactly (#172) ### Fixed diff --git a/cmd/lmm/game.go b/cmd/lmm/game.go index 7f06961..dccc7a3 100644 --- a/cmd/lmm/game.go +++ b/cmd/lmm/game.go @@ -266,8 +266,8 @@ func gameFromDetected(g steam.DetectedGame) (*domain.Game, error) { } deployMode, ok := domain.ParseDeployMode(g.DeployMode) if !ok { - return nil, fmt.Errorf("%w: steam-games.yaml: game %q: deploy_mode %q (valid: extract, copy, compile)", - domain.ErrInvalidDeployMode, g.Slug, g.DeployMode) + return nil, fmt.Errorf("%w: steam-games.yaml: game %q: deploy_mode %q (valid: %s)", + domain.ErrInvalidDeployMode, g.Slug, g.DeployMode, domain.ValidDeployModes) } return &domain.Game{ ID: g.Slug, diff --git a/internal/domain/game.go b/internal/domain/game.go index 4b9e5f7..d253916 100644 --- a/internal/domain/game.go +++ b/internal/domain/game.go @@ -22,6 +22,12 @@ func (m LinkMethod) String() string { } } +// ValidLinkMethods lists ParseLinkMethod's recognized non-empty values, in +// the same order as the type's constants, for use in "unrecognized value" +// error messages — the single source of truth so those messages can't go +// stale the way a hand-written copy did (#172 review round 1). +const ValidLinkMethods = "symlink, hardlink, copy" + // ParseLinkMethod converts a string to LinkMethod. An empty string is not // yet set and returns the default (symlink) with ok=true, so configs that // never set link_method keep working unchanged. Any other unrecognized @@ -77,6 +83,9 @@ func (m DeployMode) String() string { } } +// ValidDeployModes is ValidLinkMethods' counterpart for ParseDeployMode. +const ValidDeployModes = "extract, copy, compile" + // ParseDeployMode converts a string to DeployMode. Mirrors ParseLinkMethod's // fail-loud contract: empty keeps the default (extract) with ok=true; any // other unrecognized string returns ok=false (#172). diff --git a/internal/storage/config/config.go b/internal/storage/config/config.go index 06d91db..68b2161 100644 --- a/internal/storage/config/config.go +++ b/internal/storage/config/config.go @@ -46,8 +46,8 @@ func Load(configDir string) (*Config, error) { if cfg.LinkMethodStr != "" { method, ok := domain.ParseLinkMethod(cfg.LinkMethodStr) if !ok { - return nil, fmt.Errorf("%w: config.yaml: default_link_method %q (valid: symlink, hardlink, copy)", - domain.ErrInvalidLinkMethod, cfg.LinkMethodStr) + return nil, fmt.Errorf("%w: config.yaml: default_link_method %q (valid: %s)", + domain.ErrInvalidLinkMethod, cfg.LinkMethodStr, domain.ValidLinkMethods) } cfg.DefaultLinkMethod = method } diff --git a/internal/storage/config/config_test.go b/internal/storage/config/config_test.go index 04c29d0..9b7defd 100644 --- a/internal/storage/config/config_test.go +++ b/internal/storage/config/config_test.go @@ -148,6 +148,10 @@ games: assert.Contains(t, err.Error(), "skyrim-se") assert.Contains(t, err.Error(), "deploy_mode") assert.Contains(t, err.Error(), "compil") + // Review #172 round 1: the valid-options list had gone stale (pre-rebase + // "extract, copy" only) and silently dropped "compile" after the Icarus + // epic merge added it as a real DeployMode value. + assert.Contains(t, err.Error(), "compile", "valid-options list must include compile") } func TestSaveGame(t *testing.T) { diff --git a/internal/storage/config/games.go b/internal/storage/config/games.go index be30e93..b510cb2 100644 --- a/internal/storage/config/games.go +++ b/internal/storage/config/games.go @@ -89,13 +89,13 @@ func loadGamesLocked(configDir string) (map[string]*domain.Game, error) { for id, cfg := range gamesFile.Games { linkMethod, ok := domain.ParseLinkMethod(cfg.LinkMethod) if !ok { - return nil, fmt.Errorf("%w: games.yaml: game %q: link_method %q (valid: symlink, hardlink, copy)", - domain.ErrInvalidLinkMethod, id, cfg.LinkMethod) + return nil, fmt.Errorf("%w: games.yaml: game %q: link_method %q (valid: %s)", + domain.ErrInvalidLinkMethod, id, cfg.LinkMethod, domain.ValidLinkMethods) } deployMode, ok := domain.ParseDeployMode(cfg.DeployMode) if !ok { - return nil, fmt.Errorf("%w: games.yaml: game %q: deploy_mode %q (valid: extract, copy)", - domain.ErrInvalidDeployMode, id, cfg.DeployMode) + return nil, fmt.Errorf("%w: games.yaml: game %q: deploy_mode %q (valid: %s)", + domain.ErrInvalidDeployMode, id, cfg.DeployMode, domain.ValidDeployModes) } games[id] = &domain.Game{ ID: id, diff --git a/internal/storage/config/profiles.go b/internal/storage/config/profiles.go index 9604a7e..1f78189 100644 --- a/internal/storage/config/profiles.go +++ b/internal/storage/config/profiles.go @@ -137,8 +137,8 @@ func LoadProfile(configDir, gameID, profileName string) (*domain.Profile, error) linkMethod, ok := domain.ParseLinkMethod(cfg.LinkMethod) if !ok { - return nil, fmt.Errorf("%w: profile %q (game %q): link_method %q (valid: symlink, hardlink, copy)", - domain.ErrInvalidLinkMethod, profileName, gameID, cfg.LinkMethod) + return nil, fmt.Errorf("%w: profile %q (game %q): link_method %q (valid: %s)", + domain.ErrInvalidLinkMethod, profileName, gameID, cfg.LinkMethod, domain.ValidLinkMethods) } profile := &domain.Profile{ @@ -302,8 +302,8 @@ func ImportProfile(data []byte) (*domain.Profile, error) { linkMethod, ok := domain.ParseLinkMethod(exported.LinkMethod) if !ok { - return nil, fmt.Errorf("%w: imported profile %q (game %q): link_method %q (valid: symlink, hardlink, copy)", - domain.ErrInvalidLinkMethod, exported.Name, exported.GameID, exported.LinkMethod) + return nil, fmt.Errorf("%w: imported profile %q (game %q): link_method %q (valid: %s)", + domain.ErrInvalidLinkMethod, exported.Name, exported.GameID, exported.LinkMethod, domain.ValidLinkMethods) } p := &domain.Profile{ From 04102ec1b8445ba7d90432ecba53e74a66865248 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 15:44:21 -0400 Subject: [PATCH 45/96] docs: correct deploy-mode comment, wrap detect error (#172 review) ErrInvalidDeployMode's doc comment still hand-listed "(extract, copy)", stale after the develop merge added compile - now points at ValidDeployModes, the single source of truth, instead of a second hand-written copy. runGameDetect's gameFromDetected error returned bare, unlike its neighboring SaveGame/SaveProfile wraps in the same loop - now wrapped with step context and the game slug for consistency. Co-Authored-By: Claude Sonnet 5 --- cmd/lmm/game.go | 2 +- internal/domain/errors.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/lmm/game.go b/cmd/lmm/game.go index dccc7a3..872b4a7 100644 --- a/cmd/lmm/game.go +++ b/cmd/lmm/game.go @@ -227,7 +227,7 @@ func runGameDetect(cmd *cobra.Command, args []string) error { g := games[n-1] game, err := gameFromDetected(g) if err != nil { - return err + return fmt.Errorf("converting detected game %s: %w", g.Slug, err) } if err := config.SaveGame(svcCfg.ConfigDir, game); err != nil { return fmt.Errorf("saving game %s: %w", g.Slug, err) diff --git a/internal/domain/errors.go b/internal/domain/errors.go index a4c1137..fddfe5e 100644 --- a/internal/domain/errors.go +++ b/internal/domain/errors.go @@ -26,7 +26,7 @@ var ( // what's wrong and how to fix it (#172). ErrInvalidLinkMethod = errors.New("invalid link method") // ErrInvalidDeployMode is ErrInvalidLinkMethod's counterpart for - // deploy_mode (extract, copy) (#172). + // deploy_mode; see ValidDeployModes for the recognized names (#172). ErrInvalidDeployMode = errors.New("invalid deploy mode") ErrDependencyLoop = errors.New("circular dependency detected") ErrAuthRequired = errors.New("authentication required") From c50de320200d1b8c2ee155eab105a9d314a7d920 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 15:44:53 -0400 Subject: [PATCH 46/96] fix: honor empty NO_COLOR per presence-only spec (#112 review) --- cmd/lmm/color_test.go | 12 ++++++++++++ cmd/lmm/root.go | 5 ++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/cmd/lmm/color_test.go b/cmd/lmm/color_test.go index e71781c..5855eb5 100644 --- a/cmd/lmm/color_test.go +++ b/cmd/lmm/color_test.go @@ -69,6 +69,18 @@ func TestColorEnabled_NoColorEnvWinsOverTTY(t *testing.T) { assert.False(t, colorEnabled(), "NO_COLOR must disable color even on a real TTY") } +// TestColorEnabled_EmptyNoColorEnvStillDisables guards the no-color.org spec: +// NO_COLOR disables color when PRESENT, regardless of value - including the +// empty string. os.Getenv can't distinguish "unset" from "set to empty", so +// colorEnabled must use os.LookupEnv instead. +func TestColorEnabled_EmptyNoColorEnvStillDisables(t *testing.T) { + resetColorFlags(t) + withColorCapableStdout(t, true) + require.NoError(t, os.Setenv("NO_COLOR", "")) + + assert.False(t, colorEnabled(), "NO_COLOR set to the empty string must still disable color (presence-only semantics)") +} + func TestColorHelpers_NoOpWhenColorDisabled(t *testing.T) { resetColorFlags(t) withColorCapableStdout(t, false) diff --git a/cmd/lmm/root.go b/cmd/lmm/root.go index 780616d..d623029 100644 --- a/cmd/lmm/root.go +++ b/cmd/lmm/root.go @@ -113,7 +113,10 @@ func colorEnabled() bool { if noColor { return false } - if os.Getenv("NO_COLOR") != "" { + // Presence-only per https://no-color.org: NO_COLOR disables color when + // set to ANY value, including the empty string - os.Getenv can't tell + // "unset" from "set to empty", so this must use os.LookupEnv. + if _, set := os.LookupEnv("NO_COLOR"); set { return false } return stdoutColorCapable() From bfdf0210ce02c8e25a768a37c1bd199acf0ad986 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 15:52:11 -0400 Subject: [PATCH 47/96] fix: atomic cache commit + accurate compiler error on import (#173 review) --- internal/core/importer.go | 32 ++++--- internal/core/service_import_compile_test.go | 94 +++++++++++++++++++- 2 files changed, 115 insertions(+), 11 deletions(-) diff --git a/internal/core/importer.go b/internal/core/importer.go index cd28300..17075bb 100644 --- a/internal/core/importer.go +++ b/internal/core/importer.go @@ -116,7 +116,7 @@ func (i *Importer) Import(ctx context.Context, archivePath string, game *domain. // source pinned to check for source.Compiler, so it resolves the // game's mapped compiler-capable source from the registry instead. if i.resolveCompiler == nil { - return nil, fmt.Errorf("game %q requires DeployCompile to import %q, but no compiler-capable source is configured for this game", game.ID, filename) + return nil, fmt.Errorf("game %q requires DeployCompile to import %q, but this Importer was constructed without service context (via core.NewImporter, not Service.NewImporter) and has no compiler resolver to consult - import via the service-backed importer instead", game.ID, filename) } compiler, err := i.resolveCompiler(game.ID) if err != nil { @@ -134,7 +134,7 @@ func (i *Importer) Import(ctx context.Context, archivePath string, game *domain. } } - // Compile into a staging dir first so a mid-compile failure never + // Compile into a scratch dir first so a mid-compile failure never // leaves a partial/uncompiled artifact in the cache (mirrors #136 // review's fix for the download path's compile branch). tempDir, err := newStagingDir(i.stagingRoot, "lmm-import-compile-*") @@ -149,16 +149,28 @@ func (i *Importer) Import(ctx context.Context, archivePath string, game *domain. return nil, fmt.Errorf("compiling mod: %w", err) } - cachePath := i.cache.ModPath(game.ID, sourceID, modID, version) - // Remove existing cache if present (re-import case) - if err := os.RemoveAll(cachePath); err != nil { - return nil, fmt.Errorf("removing existing cache for re-import: %w", err) + // Stage the compiled artifact and commit it atomically (#173 review: + // mirrors Service.DownloadModToCache's compile branch). cachePath is + // never touched until commitStagedCache's single backup-then-rename + // swap, so a failure staging the artifact (or committing it) leaves + // any existing cache entry exactly as it was — unlike the previous + // remove-existing-then-copy sequence, which destroyed the prior + // entry before the copy that could still fail. + cacheMod := &domain.Mod{ID: modID, SourceID: sourceID, Version: version, GameID: game.ID} + cachePath, stagePath, err := prepareUnseededStaging(i.cache, game, cacheMod) + if err != nil { + return nil, err } - if err := os.MkdirAll(cachePath, 0755); err != nil { - return nil, fmt.Errorf("creating cache directory: %w", err) + defer os.RemoveAll(stagePath) //nolint:errcheck + + if err := os.MkdirAll(stagePath, 0755); err != nil { + return nil, fmt.Errorf("preparing cache staging: %w", err) } - if err := copyFileStreaming(compiledPath, filepath.Join(cachePath, destName)); err != nil { - return nil, fmt.Errorf("moving compiled mod to cache: %w", err) + if err := copyFileStreaming(compiledPath, filepath.Join(stagePath, destName)); err != nil { + return nil, fmt.Errorf("staging compiled mod: %w", err) + } + if err := commitStagedCache(cachePath, stagePath); err != nil { + return nil, err } fileCount = 1 } else if game.DeployMode == domain.DeployCopy { diff --git a/internal/core/service_import_compile_test.go b/internal/core/service_import_compile_test.go index 136375c..8eb0512 100644 --- a/internal/core/service_import_compile_test.go +++ b/internal/core/service_import_compile_test.go @@ -26,6 +26,29 @@ func (s *failingCompilerSource) Compile(ctx context.Context, basePakPath, source return fmt.Errorf("boom: compile always fails") } +// raceCompilerSource wraps fakeCompilerSource and, when sabotage is set, +// writes its declared output and then immediately removes it before +// returning success - deterministically reproducing "compile reported +// success, but the artifact is gone by the time it must be staged into the +// cache" without any OS-specific permission tricks. This is the shape of +// the #173 review defect: the compile step itself succeeds, but the +// subsequent step that gets the artifact into the cache can still fail. +type raceCompilerSource struct { + *fakeCompilerSource + sabotage bool +} + +func (s *raceCompilerSource) Compile(ctx context.Context, basePakPath, sourceFilePath, outputPath string) error { + s.compileCalls++ + if err := os.WriteFile(outputPath, []byte("new-content"), 0o644); err != nil { + return err + } + if s.sabotage { + return os.Remove(outputPath) + } + return nil +} + // newImportCompileTestGame builds a DeployCompile game with a registered, // game-mapped compiler source and an installed base pak - the setup #173's // import path needs to resolve a Compiler the same way @@ -295,5 +318,74 @@ func TestImportMod_DeployCompile_StandaloneImporterFailsLoud(t *testing.T) { result, err := importer.Import(context.Background(), archivePath, game, core.ImportOptions{}) require.Error(t, err) require.Nil(t, result) - require.Contains(t, err.Error(), "compiler") + // The message must name the actual cause - a standalone Importer with + // no service context, not "no compiler-capable source configured" (a + // different failure covered by TestImportMod_DeployCompile_NoCompilerSourceFailsLoud). + require.Contains(t, err.Error(), "without service context") + require.Contains(t, err.Error(), "core.NewImporter") +} + +// TestImportMod_DeployCompile_ReimportSurvivesStagingFailure pins the #173 +// review defect: the compile branch used to os.RemoveAll(cachePath) and +// then copy the compiled artifact into place, so a failure in that copy +// step destroyed a pre-existing good cache entry before ever writing its +// replacement. It now stages the compiled artifact into an isolated +// directory and commits it to cachePath atomically (commitStagedCache, +// mirroring the download path) - cachePath is never touched by a failed +// staging attempt, so a pre-existing entry must survive. +func TestImportMod_DeployCompile_ReimportSurvivesStagingFailure(t *testing.T) { + installDir := t.TempDir() + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + require.NoError(t, os.WriteFile(basePak, []byte("fake-base-pak"), 0o644)) + + cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} + svc, err := core.NewService(cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + src := &raceCompilerSource{fakeCompilerSource: &fakeCompilerSource{}} + svc.RegisterSource(src) + + game := &domain.Game{ + ID: "icarus", + InstallPath: installDir, + ModPath: t.TempDir(), + DeployMode: domain.DeployCompile, + SourceIDs: map[string]string{"fake-compiler": "external-icarus-id"}, + } + require.NoError(t, svc.AddGame(game)) + + tempDir := t.TempDir() + archivePath := filepath.Join(tempDir, "Bear_Mount.exmodz") + require.NoError(t, os.WriteFile(archivePath, []byte("good-exmodz-bytes"), 0o644)) + + opts := core.ImportOptions{SourceID: "fake-compiler", ModID: "bear-mount"} + importer := svc.NewImporter(game) + + // First import succeeds and leaves a good cache entry. + result1, err := importer.Import(context.Background(), archivePath, game, opts) + require.NoError(t, err) + + gameCache := svc.GetGameCache(game) + files, err := gameCache.ListFiles(game.ID, result1.Mod.SourceID, result1.Mod.ID, result1.Mod.Version) + require.NoError(t, err) + require.Equal(t, []string{"Bear_Mount_P.pak"}, files) + + filePath := gameCache.GetFilePath(game.ID, result1.Mod.SourceID, result1.Mod.ID, result1.Mod.Version, files[0]) + origData, err := os.ReadFile(filePath) + require.NoError(t, err) + require.Equal(t, "new-content", string(origData)) + + // Re-import the same archive; the compiler's declared output vanishes + // before it can be staged, so this import must fail... + src.sabotage = true + result2, err := importer.Import(context.Background(), archivePath, game, opts) + require.Error(t, err) + require.Nil(t, result2) + + // ...and the prior good entry must still be exactly what it was. + survivingData, err := os.ReadFile(filePath) + require.NoError(t, err, "the prior good cache entry must survive a failed re-import") + require.Equal(t, "new-content", string(survivingData)) } From 875daac88de43527aa5985a307d6052f672140de Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 16:12:27 -0400 Subject: [PATCH 48/96] fix: announce compile step instead of generic Extracting message (#190) lmm install (STRICT path) always printed "Extracting to cache..." after a mod's file(s) downloaded, even when the file was a DeployCompile .exmodz archive that gets compiled into a _P.pak rather than extracted. ApplyInstall now tracks which files actually took the compile branch (same condition DownloadModToCache itself gates on) and emits a new InstallCompiling phase per compiled file instead, carrying the source file and computed output filename so the CLI can print "Compiling -> ..." - matching the existing progress-event convention rather than inventing a new mechanism. A plain extract/copy install is unaffected. The TUI drives the same core.ApplyInstall path, so installProgressLine gained the same case for parity. --- CHANGELOG.md | 1 + cmd/lmm/install.go | 2 + cmd/lmm/install_compile_test.go | 68 ++++++++++++++++++++++ internal/core/flows.go | 43 +++++++++++++- internal/tui/service_core.go | 2 + internal/tui/service_core_internal_test.go | 11 ++++ 6 files changed, 124 insertions(+), 3 deletions(-) create mode 100644 cmd/lmm/install_compile_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index ce39491..744469f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- `lmm install` (and the TUI's equivalent progress line) announced a `.exmodz` compile step as "Extracting to cache..." — actively misleading, since compiling never extracts anything. It now prints "Compiling `` → ``..." instead, naming the actual archive and the compiled `_P.pak` output; a plain extract/copy install is unaffected and keeps today's exact "Extracting to cache..." text (#190) - `lmm import` of a local `.exmodz` file for a `deploy_mode: compile` game (Icarus) now routes through the same compile step as a download: it resolves the game's mapped `source.Compiler`-capable source from the registry and compiles the archive against the installed base pak, caching the resulting `_P.pak` the same way `DownloadModToCache` does. Previously the import path extracted/copied `.exmodz` files as-is, landing an uncompiled archive in the cache instead of a deployable pak. A missing compiler-capable source or missing base pak now fails loud with an actionable error rather than silently caching the uncompiled file; non-`.exmodz` imports are unaffected (#173) - `lmm mod disable` undeployed a mod's files and cleared `enabled`, but never cleared `deployed` — `lmm list -v` kept showing DEPLOYED yes after disable. The disable flow now clears `deployed` unconditionally after the undeploy attempt, even when the undeploy itself only partially succeeds (already a non-fatal, Note-reported condition), so the flag always reflects disable-intent rather than lagging behind a best-effort file cleanup. The symmetric enable path had the same gap — enabling a disabled mod re-deployed its files without ever setting `deployed` back to true — and is fixed the same way. Both `SetModDeployed` calls follow the same non-fatal Note convention already used by `DeployProfile`/`PurgeProfile` for this same setter: a failure to record the flag doesn't block the primary enable/disable outcome (#183) diff --git a/cmd/lmm/install.go b/cmd/lmm/install.go index 37ffe79..95bdf7f 100644 --- a/cmd/lmm/install.go +++ b/cmd/lmm/install.go @@ -614,6 +614,8 @@ func doInstall(ctx context.Context, service *core.Service, game *domain.Game, ar } case core.InstallChecksumComputed: fmt.Printf(" Checksum: %s\n", truncateChecksum(p.Detail)) + case core.InstallCompiling: + fmt.Printf("\nCompiling %s → %s...\n", displayFileLabel(*p.File), p.Detail) case core.InstallExtracting: fmt.Println("\nExtracting to cache...") case core.InstallDeploying: diff --git a/cmd/lmm/install_compile_test.go b/cmd/lmm/install_compile_test.go new file mode 100644 index 0000000..ffd18f6 --- /dev/null +++ b/cmd/lmm/install_compile_test.go @@ -0,0 +1,68 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// compilerInstallSource wraps fakeInstallSource with a source.Compiler +// implementation, so `lmm install` can drive a real DeployCompile game +// end-to-end through the CLI's exact console-output path (mirrors +// internal/core/service_icarus_compile_test.go's fakeCompilerSource, at the +// CLI layer instead of core's). +type compilerInstallSource struct { + *fakeInstallSource + compileCalls int +} + +// Compile copies the downloaded source file through unchanged - this test +// only asserts the CLI announces the compile step and uses its output, not +// that real PAK compilation happens (internal/unrealpak's own tests cover +// that). +func (s *compilerInstallSource) Compile(ctx context.Context, basePakPath, sourceFilePath, outputPath string) error { + s.compileCalls++ + data, err := os.ReadFile(sourceFilePath) + if err != nil { + return err + } + return os.WriteFile(outputPath, data, 0o644) +} + +// TestDoInstall_DeployCompile_AnnouncesCompiling guards #190 item 1: an +// install that compiles a .exmodz file must announce the compile step by +// name, not the generic "Extracting to cache..." line the plain +// extract/copy path uses (which is actively misleading here - compiling +// isn't extracting). +func TestDoInstall_DeployCompile_AnnouncesCompiling(t *testing.T) { + svc, game, src := setupDoInstallTest(t) + game.DeployMode = domain.DeployCompile + game.InstallPath = t.TempDir() + + basePak := filepath.Join(game.InstallPath, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + require.NoError(t, os.WriteFile(basePak, []byte("fake-base-pak"), 0o644)) + + compiler := &compilerInstallSource{fakeInstallSource: src} + // Re-register under the same ID so doInstall's resolved source is the + // compiler-capable wrapper, not the plain fake registered by + // setupDoInstallTest. + svc.RegisterSource(compiler) + + src.AddMod(&domain.Mod{ID: "mod1", SourceID: "test-src", Name: "Bear Mount", Version: "1.0", GameID: "g1"}, + []domain.DownloadableFile{{ID: "main", Name: "Bear Mount", FileName: "Bear_Mount.exmodz", IsPrimary: true, Category: "MAIN"}}) + src.AddDownload("main", []byte("fake-exmodz-bytes")) + + out := captureStdout(t, func() error { + return doInstall(context.Background(), svc, game, nil) + }) + + assert.Equal(t, 1, compiler.compileCalls) + assert.Contains(t, out, "Compiling Bear_Mount.exmodz → Bear_Mount_P.pak...\n") + assert.NotContains(t, out, "Extracting to cache...", "compiling isn't extracting - the generic message must not also print") +} diff --git a/internal/core/flows.go b/internal/core/flows.go index 496afab..e1b3ba0 100644 --- a/internal/core/flows.go +++ b/internal/core/flows.go @@ -668,10 +668,23 @@ const ( // succeeds). Detail carries the full (untruncated) checksum either // way; the CLI applies its own truncateChecksum. InstallChecksumComputed + // InstallCompiling fires instead of InstallExtracting, once per file, + // when a DeployCompile game's ".exmodz" file was actually compiled + // (#190 item 1) - the generic "Extracting to cache..." wording is + // misleading for a compile step, which never extracts anything. File + // identifies the source file (for displayFileLabel); Detail carries the + // compiled output filename (e.g. "Bear_Mount_P.pak"), so the CLI can + // announce "Compiling → ..." without core owning the + // exact sentence. The BATCH path never prints this (it has no + // DeployCompile support and no equivalent status line at all). + InstallCompiling // InstallExtracting mirrors doInstall's unconditional "Extracting to // cache..." status line, fired once after the STRICT-path primary's - // download(s) finish, before Install/Replace. The BATCH path never - // prints this (batchInstallMods had no equivalent status line). + // download(s) finish, before Install/Replace - unless every downloaded + // file was compiled instead (InstallCompiling fires in that case, one + // event per compiled file, and this is skipped entirely). The BATCH + // path never prints this (batchInstallMods had no equivalent status + // line). InstallExtracting // InstallDeploying mirrors "Deploying to game directory...", fired once // right before the STRICT-path primary's Install/Replace. The BATCH @@ -3924,6 +3937,13 @@ func (s *Service) applyInstallPrimary(ctx context.Context, game *domain.Game, pl var downloadedFileIDs []string var checksums []fileChecksum + // compiledFiles accumulates every file this loop actually compiled (game + // DeployCompile + a ".exmodz" file - the same condition + // DownloadModToCache itself gates on), re-derived here rather than read + // back from DownloadModToCache's result since flows.go already has + // everything the condition needs. Drives the InstallCompiling + // announcement below in place of the generic InstallExtracting one. + var compiledFiles []*domain.DownloadableFile filesTotal := len(plan.Files) for i := range plan.Files { file := &plan.Files[i] @@ -3965,9 +3985,26 @@ func (s *Service) applyInstallPrimary(ctx context.Context, game *domain.Game, pl result.FilesDeployed += downloadResult.FilesExtracted downloadedFileIDs = append(downloadedFileIDs, file.ID) + + if game.DeployMode == domain.DeployCompile && isExmodzFile(file.FileName) { + compiledFiles = append(compiledFiles, file) + } } - emit(DeployProgress{Phase: InstallExtracting, ModName: mod.Name, ModID: mod.ID}) + // A compiled file was never "extracted" - announce the compile step by + // name instead of the generic message, which is actively misleading + // here (#190 item 1). Only fires for files that actually compiled, so + // every non-DeployCompile (or non-exmodz) install keeps today's exact + // "Extracting to cache..." text unchanged. + if len(compiledFiles) > 0 { + for _, cf := range compiledFiles { + evt := base + evt.Phase, evt.File, evt.Detail = InstallCompiling, cf, compiledFileName(cf.FileName) + emit(evt) + } + } else { + emit(DeployProgress{Phase: InstallExtracting, ModName: mod.Name, ModID: mod.ID}) + } // Conflict confirmation restored to doInstall's ORIGINAL position (C1 // review finding): AFTER the primary is downloaded/extracted to cache, diff --git a/internal/tui/service_core.go b/internal/tui/service_core.go index ade32a9..6701621 100644 --- a/internal/tui/service_core.go +++ b/internal/tui/service_core.go @@ -1146,6 +1146,8 @@ func installProgressLine(modName string, p core.DeployProgress) (ActionProgress, return ActionProgress{Line: fmt.Sprintf("Installing %s: %.0f%%", modName, p.Percent), Percent: p.Percent}, true case core.InstallDepDownloading: return ActionProgress{Line: fmt.Sprintf("Installing %s: %.0f%%", p.ModName, p.Percent), Percent: p.Percent}, true + case core.InstallCompiling: + return ActionProgress{Line: fmt.Sprintf("Installing %s: compiling", modName), Percent: -1}, true case core.InstallExtracting: return ActionProgress{Line: fmt.Sprintf("Installing %s: extracting", modName), Percent: -1}, true case core.InstallDeploying: diff --git a/internal/tui/service_core_internal_test.go b/internal/tui/service_core_internal_test.go index cff011c..46fd21e 100644 --- a/internal/tui/service_core_internal_test.go +++ b/internal/tui/service_core_internal_test.go @@ -47,3 +47,14 @@ func TestSwitchProgressLine_UnhandledPhaseStillDrops(t *testing.T) { _, ok := switchProgressLine(core.DeployProgress{Phase: core.SwitchDisableNote}) assert.False(t, ok) } + +// TestInstallProgressLine_RendersCompiling guards #190 item 1's TUI parity: +// the TUI drives the exact same core.ApplyInstall/DeployProgress path as the +// CLI (installProgressLine's own doc comment), so InstallCompiling must +// compose a status line here too, not silently fall to the default case. +func TestInstallProgressLine_RendersCompiling(t *testing.T) { + line, ok := installProgressLine("Bear Mount", core.DeployProgress{Phase: core.InstallCompiling}) + assert.True(t, ok, "InstallCompiling must compose a visible progress line, not be dropped") + assert.Contains(t, line.Line, "Bear Mount") + assert.Contains(t, line.Line, "compiling") +} From 666a26229eb24f7aa4df52e572624e6c9df6ad8d Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 16:13:40 -0400 Subject: [PATCH 49/96] fix: GetEffectiveLinkMethod surfaces invalid link_method instead of silently degrading (#189) Service.GetEffectiveLinkMethod treated ANY config.LoadProfile error - missing file, unreadable, or (since #172) an invalid link_method value - as "no explicit profile override" and silently fell back to the game/global default. That was intentional for a missing/unreadable profile (profiles are optional), but meant a hand-edited profile with a typo'd link_method could deploy with the wrong method with nothing telling anyone anything was wrong - the one place #172's fail-loud contract didn't reach. GetEffectiveLinkMethod now distinguishes errors.Is(err, domain.ErrInvalidLinkMethod) from every other LoadProfile failure mode and returns it instead of degrading; the missing-file case is unchanged (still silently resolves to the game/global default). Chose error-return over a Note/warning field: every one of GetEffectiveLinkMethod's ~10 call sites (directly, or via GetInstallerForProfile's ~9 further callers) already lives inside a function returning error - DeployProfile, EnableMod/DisableMod, ApplyInstall/ApplyUpdate/ApplyRollback/ApplyImport/ApplyProfileSwitch, lmm deploy/import/install/profile apply/verify --fix - so propagating with the existing %w-wrap idiom is the smallest change that reaches every site uniformly, with no new plumbing needed. One exception: PlanInstall's conflict-detection call is a read-only preview whose existing documented policy is "ANY GetConflicts error degrades to no conflicts detected, never fails the plan" - extended that same already-established silent-degrade policy to the installer-resolution error too, rather than making a preview-only path fail loud when nothing is being written to disk. TDD: TestService_GetEffectiveLinkMethod_InvalidLinkMethodSurfaces (RED before the fix - old GetEffectiveLinkMethod had no error to check; GREEN after) plus TestService_DeployProfile_InvalidProfileLinkMethodFailsLoud, an end-to-end deploy-path proof that a bad profile now fails the deploy instead of silently writing a symlink with the game's method. TestService_GetEffectiveLinkMethod_Precedence updated to the new signature; its missing-profile-file case still asserts the game-level fallback, unchanged. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 1 + cmd/lmm/deploy.go | 6 +++- cmd/lmm/import.go | 10 ++++-- cmd/lmm/install.go | 5 ++- cmd/lmm/profile.go | 5 ++- cmd/lmm/status.go | 6 +++- cmd/lmm/verify.go | 10 ++++-- internal/core/flows.go | 69 +++++++++++++++++++++++++++++-------- internal/core/flows_test.go | 58 ++++++++++++++++++++++++++++++- internal/core/service.go | 34 +++++++++++++----- 10 files changed, 173 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce39491..b9a9eff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- `Service.GetEffectiveLinkMethod` — the profile > game > global resolution behind every deploy/install/import/status/verify operation — no longer silently swallows an invalid profile `link_method`: since #172, `config.LoadProfile` fails loud on an unrecognized value, but `GetEffectiveLinkMethod` treated ANY profile-load error, including that one, as "no explicit override" and fell back to the game/global default with nothing surfaced — meaning a hand-edited profile with a typo'd `link_method` could deploy with the wrong method, no error, no warning. It now distinguishes that case (`errors.Is(err, domain.ErrInvalidLinkMethod)`) from a missing/unreadable profile file — which still degrades silently by design, since profiles are optional — and returns the validation error instead, propagated through every call site (`DeployProfile`, `EnableMod`/`DisableMod`, `ApplyInstall`/`ApplyUpdate`/`ApplyRollback`/`ApplyImport`/`ApplyProfileSwitch`, `lmm deploy`/`import`/`install`/`profile apply`/`verify --fix`). Narrow in practice — both save and load paths validate now, so only a profile hand-edited after the fact can trigger it — but it closes the one place #172's fail-loud contract didn't reach (#189) - `lmm import` of a local `.exmodz` file for a `deploy_mode: compile` game (Icarus) now routes through the same compile step as a download: it resolves the game's mapped `source.Compiler`-capable source from the registry and compiles the archive against the installed base pak, caching the resulting `_P.pak` the same way `DownloadModToCache` does. Previously the import path extracted/copied `.exmodz` files as-is, landing an uncompiled archive in the cache instead of a deployable pak. A missing compiler-capable source or missing base pak now fails loud with an actionable error rather than silently caching the uncompiled file; non-`.exmodz` imports are unaffected (#173) - `lmm mod disable` undeployed a mod's files and cleared `enabled`, but never cleared `deployed` — `lmm list -v` kept showing DEPLOYED yes after disable. The disable flow now clears `deployed` unconditionally after the undeploy attempt, even when the undeploy itself only partially succeeds (already a non-fatal, Note-reported condition), so the flag always reflects disable-intent rather than lagging behind a best-effort file cleanup. The symmetric enable path had the same gap — enabling a disabled mod re-deployed its files without ever setting `deployed` back to true — and is fixed the same way. Both `SetModDeployed` calls follow the same non-fatal Note convention already used by `DeployProfile`/`PurgeProfile` for this same setter: a failure to record the flag doesn't block the primary enable/disable outcome (#183) diff --git a/cmd/lmm/deploy.go b/cmd/lmm/deploy.go index 461effb..280f6e1 100644 --- a/cmd/lmm/deploy.go +++ b/cmd/lmm/deploy.go @@ -93,7 +93,11 @@ func doDeploy(ctx context.Context, service *core.Service, game *domain.Game, arg if linkMethodOverride != nil { methodName = linkMethodOverride.String() } else { - methodName = service.GetEffectiveLinkMethod(game, profileName).String() + method, err := service.GetEffectiveLinkMethod(game, profileName) + if err != nil { + return err + } + methodName = method.String() } opts := core.DeployOptions{ diff --git a/cmd/lmm/import.go b/cmd/lmm/import.go index 789b618..bf0a133 100644 --- a/cmd/lmm/import.go +++ b/cmd/lmm/import.go @@ -222,7 +222,10 @@ func doImport(ctx context.Context, cmd *cobra.Command, service *core.Service, ga // Set up installer for conflict checking and deployment. The installer is // built from the already-resolved method so both stay consistent (and the // profile file is only read once). - linkMethod := service.GetEffectiveLinkMethod(game, profileName) + linkMethod, err := service.GetEffectiveLinkMethod(game, profileName) + if err != nil { + return err + } installer := service.NewInstallerWithLinker(game, service.GetLinker(linkMethod)) // Check for conflicts (unless --force) @@ -577,7 +580,10 @@ func runImportScan(cmd *cobra.Command, game *domain.Game, service *core.Service, } // Import each untracked mod - linkMethod := service.GetEffectiveLinkMethod(game, profileName) + linkMethod, err := service.GetEffectiveLinkMethod(game, profileName) + if err != nil { + return err + } // Get current installed mods for duplicate checking currentMods, _ := service.GetInstalledMods(game.ID, profileName) diff --git a/cmd/lmm/install.go b/cmd/lmm/install.go index 37ffe79..4bda721 100644 --- a/cmd/lmm/install.go +++ b/cmd/lmm/install.go @@ -1022,7 +1022,10 @@ func batchInstallMods(ctx context.Context, service *core.Service, game *domain.G } } - linkMethod := service.GetEffectiveLinkMethod(game, profileName) + linkMethod, err := service.GetEffectiveLinkMethod(game, profileName) + if err != nil { + return err + } // Set up hooks hookRunner := getHookRunner(service) diff --git a/cmd/lmm/profile.go b/cmd/lmm/profile.go index 1392c5f..f1d6048 100644 --- a/cmd/lmm/profile.go +++ b/cmd/lmm/profile.go @@ -995,7 +995,10 @@ func doProfileApply(ctx context.Context, service *core.Service, game *domain.Gam } } - installer := service.GetInstallerForProfile(game, profileName) + installer, err := service.GetInstallerForProfile(game, profileName) + if err != nil { + return err + } // Disable mods for _, im := range toDisable { diff --git a/cmd/lmm/status.go b/cmd/lmm/status.go index 5eb7d7f..c6282e8 100644 --- a/cmd/lmm/status.go +++ b/cmd/lmm/status.go @@ -237,7 +237,11 @@ func showGameStatusJSON(service *core.Service, gameID string) error { if defaultProfile, err := pm.GetDefault(gameID); err == nil { // Mirror the text twin (showGameStatus): the effective method is the // active profile's resolution (profile > game > global, #155). - out.EffectiveLinkMethod = service.GetEffectiveLinkMethod(game, defaultProfile.Name).String() + method, err := service.GetEffectiveLinkMethod(game, defaultProfile.Name) + if err != nil { + return fmt.Errorf("resolving effective link method: %w", err) + } + out.EffectiveLinkMethod = method.String() if defaultProfile.LinkMethodExplicit { out.LinkMethodSource = "profile" } diff --git a/cmd/lmm/verify.go b/cmd/lmm/verify.go index ae235c9..256ab5a 100644 --- a/cmd/lmm/verify.go +++ b/cmd/lmm/verify.go @@ -1132,9 +1132,15 @@ func repairSiblingProfiles(cmd *cobra.Command, svc *core.Service, game *domain.G // SUCCESSFUL install, likewise non-fatal: the deployment itself is fixed, // only the recorded method is stale, and failing the whole repair over it // would misreport a fixed deployment as broken. installErr and recordErr -// are mutually exclusive. +// are mutually exclusive. A GetEffectiveLinkMethod failure (#189: an invalid +// profile link_method) is reported as installErr too - it happens before +// anything is touched, same as any other reason no method could be +// resolved to install with. func relinkDeployedRow(cmd *cobra.Command, svc *core.Service, game *domain.Game, profileName string, mod *domain.InstalledMod) (installErr, recordErr, undeployErr error) { - method := svc.GetEffectiveLinkMethod(game, profileName) + method, err := svc.GetEffectiveLinkMethod(game, profileName) + if err != nil { + return err, nil, nil + } installer := svc.NewInstallerWithLinker(game, svc.GetLinker(method)) // Undeploy-then-install, the same shape DeployProfile uses // (internal/core/flows.go) and for the same reason: dst still holds diff --git a/internal/core/flows.go b/internal/core/flows.go index 496afab..c28bc44 100644 --- a/internal/core/flows.go +++ b/internal/core/flows.go @@ -72,7 +72,10 @@ func (s *Service) EnableMod(ctx context.Context, game *domain.Game, profileName, return nil, fmt.Errorf("mod not found in cache - try reinstalling with 'lmm install --id %s'", modID) } - installer := s.GetInstallerForProfile(game, profileName) + installer, err := s.GetInstallerForProfile(game, profileName) + if err != nil { + return nil, err + } if err := installer.Install(ctx, game, &mod.Mod, profileName); err != nil { return nil, fmt.Errorf("failed to deploy mod: %w", err) } @@ -141,7 +144,10 @@ func (s *Service) DisableMod(ctx context.Context, game *domain.Game, profileName } result := &DisableResult{} - installer := s.GetInstallerForProfile(game, profileName) + installer, err := s.GetInstallerForProfile(game, profileName) + if err != nil { + return nil, err + } if err := installer.Uninstall(ctx, game, &mod.Mod, profileName); err != nil { // Non-fatal — see doc comment. Historical "Warning: " prefix baked // into the text itself, matching UninstallResult's own convention. @@ -247,7 +253,10 @@ func (s *Service) UninstallMod(ctx context.Context, game *domain.Game, profileNa result.Warnings = append(result.Warnings, fmt.Sprintf("uninstall.before_each hook failed (forced): %v", err)) } - installer := s.GetInstallerForProfile(game, profileName) + installer, err := s.GetInstallerForProfile(game, profileName) + if err != nil { + return result, err + } if err := installer.Uninstall(ctx, game, &mod.Mod, profileName); err != nil { // Non-fatal - files may have been manually removed. Always // recorded; the historical "Warning: " prefix is baked into the @@ -1642,7 +1651,11 @@ func (s *Service) DeployProfile(ctx context.Context, game *domain.Game, profileN if opts.LinkMethod != nil { linkMethod = *opts.LinkMethod } else { - linkMethod = s.GetEffectiveLinkMethod(game, profileName) + method, err := s.GetEffectiveLinkMethod(game, profileName) + if err != nil { + return result, err + } + linkMethod = method } installer := s.NewInstallerWithLinker(game, s.GetLinker(linkMethod)) @@ -1961,7 +1974,10 @@ func (s *Service) purgeMods(ctx context.Context, game *domain.Game, profileName spec.emit(DeployProgress{Phase: DeployBeforeAllForced, Detail: msg}) } - installer := s.GetInstallerForProfile(game, profileName) + installer, err := s.GetInstallerForProfile(game, profileName) + if err != nil { + return err + } spec.emit(DeployProgress{Phase: DeployPurging, Total: len(mods)}) // deferredWarnings holds uninstall.after_each (per mod, in loop order) @@ -2377,8 +2393,14 @@ func (s *Service) ApplyProfileSwitch(ctx context.Context, game *domain.Game, pla // link methods - the disable loop undeploys the FROM profile's // deployments (which were made with plan.From's method), while the // enable and install loops deploy into plan.To. - fromInstaller := s.GetInstallerForProfile(game, plan.From) - toInstaller := s.GetInstallerForProfile(game, plan.To) + fromInstaller, err := s.GetInstallerForProfile(game, plan.From) + if err != nil { + return result, err + } + toInstaller, err := s.GetInstallerForProfile(game, plan.To) + if err != nil { + return result, err + } pm := s.NewProfileManager() totalDisable := len(plan.ToDisable) @@ -2830,9 +2852,16 @@ func (s *Service) PlanInstall(ctx context.Context, game *domain.Game, profileNam // Conflict detection mirrors confirmInstallConflicts exactly: ANY // GetConflicts error - including "mod not in cache" for a mod PlanInstall // has (by construction) never downloaded - degrades to "no conflicts - // detected", never fails the plan. See Conflicts' doc comment. - if conflicts, err := s.GetInstallerForProfile(game, profileName).GetConflicts(ctx, game, mod, profileName); err == nil { - plan.Conflicts = conflicts + // detected", never fails the plan. See Conflicts' doc comment. Extended + // (#189) to GetInstallerForProfile's own error (e.g. an invalid + // profile link_method): this is a read-only preview, not a deploy, so + // the existing "never fails the plan" policy still applies - the + // installer's resolution simply doesn't get to say whether this + // specific plan conflicts with anything already on disk. + if installer, err := s.GetInstallerForProfile(game, profileName); err == nil { + if conflicts, err := installer.GetConflicts(ctx, game, mod, profileName); err == nil { + plan.Conflicts = conflicts + } } return plan, nil @@ -3523,7 +3552,10 @@ func (s *Service) ApplyInstall(ctx context.Context, game *domain.Game, plan *Ins emit(DeployProgress{Phase: InstallBeforeAllForced, Detail: msg}) } - linkMethod := s.GetEffectiveLinkMethod(game, plan.Profile) + linkMethod, err := s.GetEffectiveLinkMethod(game, plan.Profile) + if err != nil { + return result, err + } pm := s.NewProfileManager() // deferredWarnings holds every install.after_each (BATCH path: every mod @@ -4326,7 +4358,10 @@ func (s *Service) ApplyUpdate(ctx context.Context, game *domain.Game, profileNam emit(evt) } - linkMethod := s.GetEffectiveLinkMethod(game, profileName) + linkMethod, err := s.GetEffectiveLinkMethod(game, profileName) + if err != nil { + return result, err + } installer := s.NewInstallerWithLinker(game, s.GetLinker(linkMethod)) hookCtx.ModID, hookCtx.ModName, hookCtx.ModVersion = newMod.ID, newMod.Name, newMod.Version @@ -4567,7 +4602,10 @@ func (s *Service) ApplyRollback(ctx context.Context, game *domain.Game, profileN emit(evt) } - linkMethod := s.GetEffectiveLinkMethod(game, profileName) + linkMethod, err := s.GetEffectiveLinkMethod(game, profileName) + if err != nil { + return result, err + } installer := s.NewInstallerWithLinker(game, s.GetLinker(linkMethod)) prevMod := mod.Mod @@ -4898,7 +4936,10 @@ func (s *Service) ApplyImport(ctx context.Context, game *domain.Game, plan *Impo return result, nil } - installer := s.GetInstallerForProfile(game, profile.Name) + installer, err := s.GetInstallerForProfile(game, profile.Name) + if err != nil { + return result, err + } total := len(toDownload) emit(DeployProgress{Phase: ImportInstalling, Total: total}) diff --git a/internal/core/flows_test.go b/internal/core/flows_test.go index d2239f4..87b40ed 100644 --- a/internal/core/flows_test.go +++ b/internal/core/flows_test.go @@ -1156,11 +1156,37 @@ func TestService_GetEffectiveLinkMethod_Precedence(t *testing.T) { } } - assert.Equal(t, tc.expectedMethod, svc.GetEffectiveLinkMethod(game, "default")) + method, err := svc.GetEffectiveLinkMethod(game, "default") + require.NoError(t, err) + assert.Equal(t, tc.expectedMethod, method) }) } } +// TestService_GetEffectiveLinkMethod_InvalidLinkMethodSurfaces pins #189: a +// hand-edited profile with an unrecognized link_method must surface as an +// error naming domain.ErrInvalidLinkMethod, not silently degrade to the +// game/global default the way a missing profile file does (the precedence +// test above). Without the fix, GetEffectiveLinkMethod would return the +// game's LinkHardlink with no error - the exact silent-misbehavior #172's +// fail-loud contract exists to prevent, one layer deeper on the deploy path. +func TestService_GetEffectiveLinkMethod_InvalidLinkMethodSurfaces(t *testing.T) { + svc := newFlowsTestService(t) + game := &domain.Game{ID: "g1", Name: "Game", ModPath: t.TempDir(), LinkMethod: domain.LinkHardlink, LinkMethodExplicit: true} + + _, err := svc.NewProfileManager().Create("g1", "default") + require.NoError(t, err) + profilePath := filepath.Join(svc.ConfigDir(), "games", "g1", "profiles", "default.yaml") + data, err := os.ReadFile(profilePath) + require.NoError(t, err) + require.NoError(t, os.WriteFile(profilePath, append(data, []byte("link_method: bogus\n")...), 0644)) + + _, err = svc.GetEffectiveLinkMethod(game, "default") + + require.Error(t, err) + assert.ErrorIs(t, err, domain.ErrInvalidLinkMethod) +} + func linkMethodPtr(m domain.LinkMethod) *domain.LinkMethod { return &m } @@ -1214,6 +1240,36 @@ func TestService_DeployProfile_CLIMethodOverrideBeatsProfileLinkMethod(t *testin assert.NotEqual(t, os.FileMode(0), info.Mode()&os.ModeSymlink, "--method symlink must beat the profile's copy") } +// TestService_DeployProfile_InvalidProfileLinkMethodFailsLoud is #189's +// deploy-path-level proof, one layer up from +// TestService_GetEffectiveLinkMethod_InvalidLinkMethodSurfaces: a profile +// with a hand-edited, unrecognized link_method must VISIBLY fail the deploy +// rather than silently deploying with the game's method. Before the fix, +// this would have deployed 1 mod with a real symlink at gameDir/plugin.esp; +// after, DeployProfile errors and nothing is written. +func TestService_DeployProfile_InvalidProfileLinkMethodFailsLoud(t *testing.T) { + svc := newFlowsTestService(t) + gameDir := t.TempDir() + game := &domain.Game{ID: "g1", Name: "Game", ModPath: gameDir, LinkMethod: domain.LinkSymlink, LinkMethodExplicit: true} + + seedInstalledMod(t, svc, game, "src", "1", "1.0", true, map[string][]byte{"plugin.esp": []byte("data")}) + seedProfileWithMod(t, svc, "g1", "default", "src", "1", "1.0") + profilePath := filepath.Join(svc.ConfigDir(), "games", "g1", "profiles", "default.yaml") + data, err := os.ReadFile(profilePath) + require.NoError(t, err) + require.NoError(t, os.WriteFile(profilePath, append(data, []byte("link_method: bogus\n")...), 0644)) + + result, err := svc.DeployProfile(context.Background(), game, "default", core.DeployOptions{}, nil) + + require.Error(t, err) + assert.ErrorIs(t, err, domain.ErrInvalidLinkMethod) + if result != nil { + assert.Zero(t, result.Deployed, "an invalid link_method must deploy nothing, not fall back to the game's method") + } + _, statErr := os.Lstat(filepath.Join(gameDir, "plugin.esp")) + assert.True(t, os.IsNotExist(statErr), "no file may be deployed when the effective link method can't be resolved") +} + // TestService_DeployProfile_PurgeRemovesFilesFirstAndPreservesEnabledSet // guards --purge's two documented behaviors. The disabled mod is the key // witness for "removed first": it is excluded from the redeploy pass diff --git a/internal/core/service.go b/internal/core/service.go index 8fab764..7979c78 100644 --- a/internal/core/service.go +++ b/internal/core/service.go @@ -1073,13 +1073,26 @@ func (s *Service) GetGameLinkMethod(game *domain.Game) domain.LinkMethod { // global default (#81). A missing or unreadable profile degrades to the // game-level resolution rather than erroring - callers resolving a method are // deploying, not validating, and the profile's absence is diagnosed elsewhere. -// The CLI --method override sits above all of these and is applied by callers +// An invalid link_method value in an otherwise-loadable profile is different: +// #172 made that a fail-loud load-time error at every explicit LoadProfile +// call site, and degrading here would deploy with the wrong method with +// nothing telling the caller anything was wrong - so that one LoadProfile +// failure mode (errors.Is domain.ErrInvalidLinkMethod) is returned as an +// error instead of folding into the silent-degrade path (#189). The CLI +// --method override sits above all of these and is applied by callers // (see DeployOptions.LinkMethod). -func (s *Service) GetEffectiveLinkMethod(game *domain.Game, profileName string) domain.LinkMethod { - if profile, err := config.LoadProfile(s.configDir, game.ID, profileName); err == nil && profile.LinkMethodExplicit { - return profile.LinkMethod +func (s *Service) GetEffectiveLinkMethod(game *domain.Game, profileName string) (domain.LinkMethod, error) { + profile, err := config.LoadProfile(s.configDir, game.ID, profileName) + if err != nil { + if errors.Is(err, domain.ErrInvalidLinkMethod) { + return 0, fmt.Errorf("resolving effective link method: %w", err) + } + return s.GetGameLinkMethod(game), nil + } + if profile.LinkMethodExplicit { + return profile.LinkMethod, nil } - return s.GetGameLinkMethod(game) + return s.GetGameLinkMethod(game), nil } // GetInstaller returns an Installer configured for the given game @@ -1089,9 +1102,14 @@ func (s *Service) GetInstaller(game *domain.Game) *Installer { // GetInstallerForProfile returns an Installer whose linker honors // profileName's effective link method (GetEffectiveLinkMethod) - the -// profile-aware companion to GetInstaller. -func (s *Service) GetInstallerForProfile(game *domain.Game, profileName string) *Installer { - return s.NewInstallerWithLinker(game, s.GetLinker(s.GetEffectiveLinkMethod(game, profileName))) +// profile-aware companion to GetInstaller. Propagates GetEffectiveLinkMethod's +// new error case (#189) rather than swallowing it. +func (s *Service) GetInstallerForProfile(game *domain.Game, profileName string) (*Installer, error) { + method, err := s.GetEffectiveLinkMethod(game, profileName) + if err != nil { + return nil, err + } + return s.NewInstallerWithLinker(game, s.GetLinker(method)), nil } // NewInstallerWithLinker returns an Installer for the given game using a From f0752225222c965949fe7643f7539dbeca953feb Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 16:14:52 -0400 Subject: [PATCH 50/96] fix: name blank CompressionMethods slots in unrealpak errors (#190) methodName() returned "" for both a genuinely out-of-range CompressionMethodIndex and an in-range slot the pak's footer table simply never named, so an unsupported-method refusal for the latter read as compression method "" (index N) - nothing a reader could act on. An in-range but blank slot now returns "unnamed method N" instead, naming the actual slot; a real out-of-range index still yields "" (no supported method matches either way, so refusal behavior is unchanged). Adjusted the one existing test that hits this path to assert the message names the slot. --- CHANGELOG.md | 1 + internal/unrealpak/reader.go | 14 ++++++++++---- internal/unrealpak/zlib_test.go | 10 +++++++++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 744469f..b25b589 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- `internal/unrealpak`'s unsupported-compression-method refusal named a blank `CompressionMethods` table slot as `compression method "" (index N)` — an in-range but never-named slot, distinct from a genuinely out-of-range index. It now names the slot itself (`unnamed method N`), so the refusal is actionable instead of pointing at an empty string (#190) - `lmm install` (and the TUI's equivalent progress line) announced a `.exmodz` compile step as "Extracting to cache..." — actively misleading, since compiling never extracts anything. It now prints "Compiling `` → ``..." instead, naming the actual archive and the compiled `_P.pak` output; a plain extract/copy install is unaffected and keeps today's exact "Extracting to cache..." text (#190) - `lmm import` of a local `.exmodz` file for a `deploy_mode: compile` game (Icarus) now routes through the same compile step as a download: it resolves the game's mapped `source.Compiler`-capable source from the registry and compiles the archive against the installed base pak, caching the resulting `_P.pak` the same way `DownloadModToCache` does. Previously the import path extracted/copied `.exmodz` files as-is, landing an uncompiled archive in the cache instead of a deployable pak. A missing compiler-capable source or missing base pak now fails loud with an actionable error rather than silently caching the uncompiled file; non-`.exmodz` imports are unaffected (#173) - `lmm mod disable` undeployed a mod's files and cleared `enabled`, but never cleared `deployed` — `lmm list -v` kept showing DEPLOYED yes after disable. The disable flow now clears `deployed` unconditionally after the undeploy attempt, even when the undeploy itself only partially succeeds (already a non-fatal, Note-reported condition), so the flag always reflects disable-intent rather than lagging behind a best-effort file cleanup. The symmetric enable path had the same gap — enabling a disabled mod re-deployed its files without ever setting `deployed` back to true — and is fixed the same way. Both `SetModDeployed` calls follow the same non-fatal Note convention already used by `DeployProfile`/`PurgeProfile` for this same setter: a failure to record the flag doesn't block the primary enable/disable outcome (#183) diff --git a/internal/unrealpak/reader.go b/internal/unrealpak/reader.go index 6ded9be..818a04d 100644 --- a/internal/unrealpak/reader.go +++ b/internal/unrealpak/reader.go @@ -75,14 +75,20 @@ func Open(path string) (*Reader, error) { } // methodName resolves a 1-based CompressionMethodIndex against this pak's own -// footer table. An index with no corresponding name yields "", which no -// supported method matches, so it falls through to the unsupported-format -// error rather than being silently treated as stored. +// footer table. An out-of-range index yields "", which no supported method +// matches, so it falls through to the unsupported-format error rather than +// being silently treated as stored. An in-range but blank slot - a valid +// table position the pak simply never named - yields a descriptive +// placeholder ("unnamed method N") instead of "", so that same refusal names +// the actual slot rather than an unhelpfully empty string. func (r *Reader) methodName(method int32) string { if method < 1 || int(method) > len(r.methods) { return "" } - return r.methods[method-1] + if name := r.methods[method-1]; name != "" { + return name + } + return fmt.Sprintf("unnamed method %d", method) } // validateAllocSize checks a length field read from pak data before it is diff --git a/internal/unrealpak/zlib_test.go b/internal/unrealpak/zlib_test.go index 78e3610..6863c8e 100644 --- a/internal/unrealpak/zlib_test.go +++ b/internal/unrealpak/zlib_test.go @@ -234,6 +234,10 @@ func TestReadFile_ZlibMultiBlock(t *testing.T) { } // An index with no name in the table is unsupported, never silently stored. +// The refusal must also be actionable (#190 item 2): a blank table slot +// used to render as a bare `compression method "" (index 3)`, which names +// nothing a reader could act on - it must instead name the slot itself +// (e.g. "unnamed method 3"). func TestReadFile_UnnamedMethodIndex_IsUnsupported(t *testing.T) { p := writeMethodPak(t, []string{"Zlib"}, []zlibFixture{ {path: "x/Y.json", blocks: [][]byte{[]byte("{}")}, method: 3}, @@ -245,9 +249,13 @@ func TestReadFile_UnnamedMethodIndex_IsUnsupported(t *testing.T) { } defer r.Close() //nolint:errcheck - if _, err := r.ReadFile("x/Y.json"); !errors.Is(err, ErrUnsupportedFormat) { + _, err = r.ReadFile("x/Y.json") + if !errors.Is(err, ErrUnsupportedFormat) { t.Fatalf("err = %v, want ErrUnsupportedFormat", err) } + if !strings.Contains(err.Error(), "unnamed method 3") { + t.Errorf("err = %q, want it to name the blank slot (e.g. %q)", err, "unnamed method 3") + } } // A corrupted compressed payload must fail the entry's SHA1 gate. From c52353069c407d4cfcfa32d1f61b53945ce538d3 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 16:17:42 -0400 Subject: [PATCH 51/96] fix: dedup symlinked Steam roots so detect only scans once (#190) FindSteamRoots checked ~/.steam/steam and ~/.local/share/Steam for existence independently, but on many real Linux installs one is a symlink to the other - both existed as candidates, so DetectGames' whole library scan ran twice against the identical real directory, doubling every warning it produced (e.g. a stale libraryfolders.vdf entry pointing at a missing install dir). FindSteamRoots now dedups candidates by their resolved (symlink-followed) real path, keeping only the first per its existing priority order; two genuinely separate roots are unaffected. --- CHANGELOG.md | 1 + internal/source/steam/steam.go | 21 +++++++++++++ internal/source/steam/steam_test.go | 49 +++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b25b589..62aab64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- `lmm game detect` printed every stale-library warning twice on Linux installs where `~/.steam/steam` is a symlink to `~/.local/share/Steam` (or the reverse): both existed as candidate Steam roots, so the whole library scan — and every warning it produced — ran twice against the identical real directory. `FindSteamRoots` now dedups candidates by their resolved (symlink-followed) real path, keeping only the first; a genuinely separate second root is unaffected (#190) - `internal/unrealpak`'s unsupported-compression-method refusal named a blank `CompressionMethods` table slot as `compression method "" (index N)` — an in-range but never-named slot, distinct from a genuinely out-of-range index. It now names the slot itself (`unnamed method N`), so the refusal is actionable instead of pointing at an empty string (#190) - `lmm install` (and the TUI's equivalent progress line) announced a `.exmodz` compile step as "Extracting to cache..." — actively misleading, since compiling never extracts anything. It now prints "Compiling `` → ``..." instead, naming the actual archive and the compiled `_P.pak` output; a plain extract/copy install is unaffected and keeps today's exact "Extracting to cache..." text (#190) - `lmm import` of a local `.exmodz` file for a `deploy_mode: compile` game (Icarus) now routes through the same compile step as a download: it resolves the game's mapped `source.Compiler`-capable source from the registry and compiles the archive against the installed base pak, caching the resulting `_P.pak` the same way `DownloadModToCache` does. Previously the import path extracted/copied `.exmodz` files as-is, landing an uncompiled archive in the cache instead of a deployable pak. A missing compiler-capable source or missing base pak now fails loud with an actionable error rather than silently caching the uncompiled file; non-`.exmodz` imports are unaffected (#173) diff --git a/internal/source/steam/steam.go b/internal/source/steam/steam.go index 852fc7b..5eb8c2f 100644 --- a/internal/source/steam/steam.go +++ b/internal/source/steam/steam.go @@ -20,6 +20,14 @@ type DetectedGame struct { } // FindSteamRoots returns candidate Steam installation roots in search order. +// On many real Linux installs ~/.steam/steam is a symlink to +// ~/.local/share/Steam (or the reverse) - both paths exist and both pass the +// existence check below, but they are the same real directory. Scanning both +// would run DetectGames' whole library scan twice against identical data, +// duplicating every warning it produces (and doing twice the redundant +// work). Resolved-path dedup keeps only the first candidate (this list's own +// priority order) whenever a later one turns out to be the same real +// directory as one already kept. func FindSteamRoots() []string { home, _ := os.UserHomeDir() candidates := []string{ @@ -30,6 +38,7 @@ func FindSteamRoots() []string { candidates = append([]string{p}, candidates...) } var out []string + seenReal := make(map[string]bool) for _, p := range candidates { if p == "" { continue @@ -38,6 +47,18 @@ func FindSteamRoots() []string { if err != nil || !info.IsDir() { continue } + // realPath falls back to p itself if it can't be resolved (e.g. a + // permission error mid-resolution) - the existence check above + // already confirmed p is a real, statable directory, so it is + // never silently dropped. + realPath, err := filepath.EvalSymlinks(p) + if err != nil { + realPath = p + } + if seenReal[realPath] { + continue + } + seenReal[realPath] = true out = append(out, p) } return out diff --git a/internal/source/steam/steam_test.go b/internal/source/steam/steam_test.go index fc8aaee..3e5ae14 100644 --- a/internal/source/steam/steam_test.go +++ b/internal/source/steam/steam_test.go @@ -144,6 +144,27 @@ func TestFindSteamRoots_STEAMROOTEnv_NonexistentPathSkipped(t *testing.T) { assert.Empty(t, FindSteamRoots()) } +// TestFindSteamRoots_SymlinkedDuplicate_ReturnsOnlyOne guards #190 item 3: +// on many real Linux Steam installs, ~/.steam/steam is a symlink to +// ~/.local/share/Steam - both candidate paths exist and both pass FindSteamRoots' +// existence check, but they are the SAME real directory. Returning both +// made DetectGames scan (and warn about) that one real library twice. Since +// the two roots resolve to the same real path, only the first (".steam/steam", +// this package's own priority order) should survive. +func TestFindSteamRoots_SymlinkedDuplicate_ReturnsOnlyOne(t *testing.T) { + home := t.TempDir() + realSteam := filepath.Join(home, ".local", "share", "Steam") + require.NoError(t, os.MkdirAll(realSteam, 0755)) + require.NoError(t, os.MkdirAll(filepath.Join(home, ".steam"), 0755)) + require.NoError(t, os.Symlink(realSteam, filepath.Join(home, ".steam", "steam"))) + t.Setenv("HOME", home) + t.Setenv("STEAM_ROOT", "") + + roots := FindSteamRoots() + assert.Equal(t, []string{filepath.Join(home, ".steam", "steam")}, roots, + "a symlinked duplicate of an already-listed root must not appear twice") +} + // --- DetectGames (steam.go), against a fabricated library tree --- // writeAppManifest writes a minimal appmanifest_.acf into steamapps. @@ -224,6 +245,34 @@ func TestDetectGames_MissingInstallDir_Warns(t *testing.T) { assert.Contains(t, warnings[0], "install dir missing") } +// TestDetectGames_SymlinkedDuplicateRoot_WarnsOnce is the end-to-end guard +// for #190 item 3 at the reported symptom's own level: `lmm game detect` +// printed each stale-library warning twice against a real Linux install +// where ~/.steam/steam symlinks to ~/.local/share/Steam - both roots exist, +// so DetectGames' library scan (and every warning it produces) used to run +// twice against the identical, real directory. FindSteamRoots' resolved-path +// dedup means the second, symlinked root never reaches the scan at all. +func TestDetectGames_SymlinkedDuplicateRoot_WarnsOnce(t *testing.T) { + home := t.TempDir() + realSteam := filepath.Join(home, ".local", "share", "Steam") + steamapps := filepath.Join(realSteam, "steamapps") + require.NoError(t, os.MkdirAll(steamapps, 0755)) + require.NoError(t, os.MkdirAll(filepath.Join(home, ".steam"), 0755)) + require.NoError(t, os.Symlink(realSteam, filepath.Join(home, ".steam", "steam"))) + t.Setenv("HOME", home) + t.Setenv("STEAM_ROOT", "") + + // Deliberately no steamapps/common/ directory - the same + // "stale library" shape TestDetectGames_MissingInstallDir_Warns uses. + writeAppManifest(t, steamapps, "489830", "Skyrim Special Edition") + + games, warnings, err := DetectGames(t.TempDir()) + require.NoError(t, err) + assert.Empty(t, games) + require.Len(t, warnings, 1, "a symlinked duplicate root must not double every warning: got %v", warnings) + assert.Contains(t, warnings[0], "install dir missing") +} + func TestDetectGames_DedupsSameGameAcrossLibraries(t *testing.T) { home := t.TempDir() t.Setenv("HOME", home) From 717b0f7ddc286abbd3c55198839657716f1ba777 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 16:22:44 -0400 Subject: [PATCH 52/96] fix: remove now-empty per-mod cache dir after last version deleted (#190) Cache.Delete removed a mod's version directory but left its container (-/) behind once every version was gone - permanent, accumulating litter. Delete now also removes that container, but only when it's actually empty; a mod with other cached versions still under it, or an already-nonexistent container, is left untouched either way. Every existing caller (uninstall, reinstall/replace paths) shares this fix automatically since they all route through the same method; reinstall paths recreate the container via their own MkdirAll immediately after, so the transient removal is a no-op there. No locking added (this package has no concurrent-write story today - a plain read-then-remove matches every other cache directory operation here); --keep-cache is unaffected since Delete is never called at all in that case. --- CHANGELOG.md | 1 + cmd/lmm/uninstall_cache_test.go | 74 ++++++++++++++++++++++++++++ internal/storage/cache/cache.go | 30 ++++++++++- internal/storage/cache/cache_test.go | 60 ++++++++++++++++++++++ 4 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 cmd/lmm/uninstall_cache_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 62aab64..b63091b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- `lmm uninstall` left behind the now-empty per-mod cache directory (`-/`) after removing a mod's last cached version — cosmetic litter that accumulated indefinitely. `Cache.Delete` now removes the container too, but only when it's actually empty (a mod with other cached versions, or `--keep-cache`, is unaffected) (#190) - `lmm game detect` printed every stale-library warning twice on Linux installs where `~/.steam/steam` is a symlink to `~/.local/share/Steam` (or the reverse): both existed as candidate Steam roots, so the whole library scan — and every warning it produced — ran twice against the identical real directory. `FindSteamRoots` now dedups candidates by their resolved (symlink-followed) real path, keeping only the first; a genuinely separate second root is unaffected (#190) - `internal/unrealpak`'s unsupported-compression-method refusal named a blank `CompressionMethods` table slot as `compression method "" (index N)` — an in-range but never-named slot, distinct from a genuinely out-of-range index. It now names the slot itself (`unnamed method N`), so the refusal is actionable instead of pointing at an empty string (#190) - `lmm install` (and the TUI's equivalent progress line) announced a `.exmodz` compile step as "Extracting to cache..." — actively misleading, since compiling never extracts anything. It now prints "Compiling `` → ``..." instead, naming the actual archive and the compiled `_P.pak` output; a plain extract/copy install is unaffected and keeps today's exact "Extracting to cache..." text (#190) diff --git a/cmd/lmm/uninstall_cache_test.go b/cmd/lmm/uninstall_cache_test.go new file mode 100644 index 0000000..838f304 --- /dev/null +++ b/cmd/lmm/uninstall_cache_test.go @@ -0,0 +1,74 @@ +package main + +import ( + "context" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/core" + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// modCacheDir returns the per-mod cache container directory (the version +// directory's parent) that #190 item 4's cleanup must remove once empty. +func modCacheDir(svc *core.Service, game *domain.Game, sourceID, modID, version string) string { + return filepath.Dir(svc.GetGameCache(game).ModPath(game.ID, sourceID, modID, version)) +} + +// resetUninstallFlags saves and resets uninstall's package-level flag +// globals for a test using setupDoDeployTest's fixture (which resets +// deploy's own globals, not uninstall's) to drive doUninstall directly. +func resetUninstallFlags(t *testing.T) { + t.Helper() + oldSource, oldProfile, oldKeep, oldForce := uninstallSource, uninstallProfile, uninstallKeep, uninstallForce + uninstallSource = "" + uninstallProfile = "" + uninstallKeep = false + uninstallForce = false + t.Cleanup(func() { + uninstallSource, uninstallProfile, uninstallKeep, uninstallForce = oldSource, oldProfile, oldKeep, oldForce + }) +} + +// TestDoUninstall_RemovesNowEmptyModCacheDirectory guards #190 item 4: +// uninstalling a mod's only cached version left the empty per-mod cache +// directory (-/) behind. It must be removed, but only +// because it's now empty - the underlying behavior lives in +// internal/storage/cache.Cache.Delete, exercised here end-to-end through +// the real CLI command. +func TestDoUninstall_RemovesNowEmptyModCacheDirectory(t *testing.T) { + svc, game := setupDoDeployTest(t) + resetUninstallFlags(t) + seedDeployableMod(t, svc, game, "a", "Mod A", "a.esp") + + dir := modCacheDir(svc, game, "src", "a", "1.0") + require.DirExists(t, dir) + + _ = captureStdout(t, func() error { + return doUninstall(context.Background(), svc, game, "a") + }) + + assert.NoDirExists(t, dir, "the now-empty per-mod cache directory must not be left behind") +} + +// TestDoUninstall_KeepCache_LeavesModCacheDirectoryIntact is the negative +// case: --keep-cache must never trigger the cleanup, matching its own +// contract of preserving cached files for reinstallation. +func TestDoUninstall_KeepCache_LeavesModCacheDirectoryIntact(t *testing.T) { + svc, game := setupDoDeployTest(t) + resetUninstallFlags(t) + seedDeployableMod(t, svc, game, "a", "Mod A", "a.esp") + + uninstallKeep = true + + dir := modCacheDir(svc, game, "src", "a", "1.0") + require.DirExists(t, dir) + + _ = captureStdout(t, func() error { + return doUninstall(context.Background(), svc, game, "a") + }) + + assert.DirExists(t, dir, "--keep-cache must preserve the cache directory entirely") +} diff --git a/internal/storage/cache/cache.go b/internal/storage/cache/cache.go index c0ce26f..b6d3b18 100644 --- a/internal/storage/cache/cache.go +++ b/internal/storage/cache/cache.go @@ -321,15 +321,43 @@ func walkEntries(modPath string, includeReserved bool) ([]string, error) { return files, nil } -// Delete removes a cached mod version +// Delete removes a cached mod version, then removes the mod's per-mod +// container directory (ModPath's parent - the "-" directory +// version subdirectories live under) if this was its last remaining +// version. The container has no meaning once nothing is left under it, so +// leaving it behind after every version is gone is just litter (#190 item +// 4). A container that still holds another version, or that never existed, +// is left alone either way - never an error. func (c *Cache) Delete(gameID, sourceID, modID, version string) error { modPath := c.ModPath(gameID, sourceID, modID, version) if err := os.RemoveAll(modPath); err != nil { return fmt.Errorf("deleting cached mod: %w", err) } + if err := removeIfEmpty(filepath.Dir(modPath)); err != nil { + return fmt.Errorf("removing empty mod cache directory: %w", err) + } return nil } +// removeIfEmpty removes dir if it exists and holds no entries. A dir that +// doesn't exist, or still has content, is left untouched - both are normal, +// not errors. No locking: this package has no concurrent-write story today +// (single Service, sequential cache mutations), so this is a plain +// read-then-remove, same as every other cache directory operation here. +func removeIfEmpty(dir string) error { + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if len(entries) > 0 { + return nil + } + return os.Remove(dir) +} + // GetFilePath returns the full path to a cached file func (c *Cache) GetFilePath(gameID, sourceID, modID, version, relativePath string) string { return filepath.Join(c.ModPath(gameID, sourceID, modID, version), relativePath) diff --git a/internal/storage/cache/cache_test.go b/internal/storage/cache/cache_test.go index f8f658c..e680be5 100644 --- a/internal/storage/cache/cache_test.go +++ b/internal/storage/cache/cache_test.go @@ -218,6 +218,66 @@ func TestCache_Delete(t *testing.T) { assert.False(t, c.Exists("skyrim-se", "nexusmods", "12345", "1.0.0")) } +// TestCache_Delete_RemovesNowEmptyModDirectory guards #190 item 4: deleting +// a mod's only cached version used to leave the empty per-mod container +// directory (/-/) behind as litter. Delete now +// removes it too, but only once nothing is left under it. +func TestCache_Delete_RemovesNowEmptyModDirectory(t *testing.T) { + dir := t.TempDir() + c := cache.New(dir) + + require.NoError(t, c.Store("skyrim-se", "nexusmods", "12345", "1.0.0", "test.txt", []byte("data"))) + modDir := filepath.Join(dir, "skyrim-se", "nexusmods-12345") + require.DirExists(t, modDir) + + require.NoError(t, c.Delete("skyrim-se", "nexusmods", "12345", "1.0.0")) + assert.NoDirExists(t, modDir, "the now-empty per-mod cache directory must not be left behind") +} + +// TestCache_Delete_KeepsModDirectoryWhenOtherVersionsRemain is the negative +// case: a mod with more than one cached version must keep its container +// (and the other version untouched) after deleting just one. +func TestCache_Delete_KeepsModDirectoryWhenOtherVersionsRemain(t *testing.T) { + dir := t.TempDir() + c := cache.New(dir) + + require.NoError(t, c.Store("skyrim-se", "nexusmods", "12345", "1.0.0", "old.txt", []byte("old"))) + require.NoError(t, c.Store("skyrim-se", "nexusmods", "12345", "2.0.0", "new.txt", []byte("new"))) + modDir := filepath.Join(dir, "skyrim-se", "nexusmods-12345") + + require.NoError(t, c.Delete("skyrim-se", "nexusmods", "12345", "1.0.0")) + assert.DirExists(t, modDir, "a sibling version still lives here - the container must survive") + assert.True(t, c.Exists("skyrim-se", "nexusmods", "12345", "2.0.0"), "the other version must be untouched") +} + +// TestCache_Delete_NonexistentVersion_NoopsCleanly: deleting a version that +// was never cached (e.g. --keep-cache preserved it but a later plain delete +// targets an already-gone entry) must not error just because there's no +// container to clean up either. +func TestCache_Delete_NonexistentVersion_NoopsCleanly(t *testing.T) { + dir := t.TempDir() + c := cache.New(dir) + + err := c.Delete("skyrim-se", "nexusmods", "12345", "1.0.0") + assert.NoError(t, err) +} + +// TestCache_Delete_GameScoped_RemovesNowEmptyModDirectory: the game-scoped +// layout (per-game cache_path override) omits the gameID level, but the +// empty-container cleanup is purely path-relative, so it must apply there +// too. +func TestCache_Delete_GameScoped_RemovesNowEmptyModDirectory(t *testing.T) { + dir := t.TempDir() + c := cache.NewGameScoped(dir) + + require.NoError(t, c.Store("starrupture", "nexusmods", "35", "1.00", "file.pak", []byte("data"))) + modDir := filepath.Join(dir, "nexusmods-35") + require.DirExists(t, modDir) + + require.NoError(t, c.Delete("starrupture", "nexusmods", "35", "1.00")) + assert.NoDirExists(t, modDir) +} + func TestCache_Exists_ListFiles_GameScoped(t *testing.T) { dir := t.TempDir() c := cache.NewGameScoped(dir) From 2ffa683b97ad728239446a7bdfc4aec29368e9e3 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 16:31:26 -0400 Subject: [PATCH 53/96] fix: drop duplicated error wrap in status (#189 review) GetEffectiveLinkMethod already wraps its ErrInvalidLinkMethod case with "resolving effective link method: %w" (service.go), and the underlying LoadProfile error already carries the profile/game/field/value context - status.go's outer wrap added the same prefix again with no distinct information. Return the error bare, matching every other GetEffectiveLinkMethod caller (deploy.go, import.go x2, install.go). Co-Authored-By: Claude Sonnet 5 --- cmd/lmm/status.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lmm/status.go b/cmd/lmm/status.go index c6282e8..9050868 100644 --- a/cmd/lmm/status.go +++ b/cmd/lmm/status.go @@ -239,7 +239,7 @@ func showGameStatusJSON(service *core.Service, gameID string) error { // active profile's resolution (profile > game > global, #155). method, err := service.GetEffectiveLinkMethod(game, defaultProfile.Name) if err != nil { - return fmt.Errorf("resolving effective link method: %w", err) + return err } out.EffectiveLinkMethod = method.String() if defaultProfile.LinkMethodExplicit { From 6b891491eb20993539edbe02eceaf11455af63b8 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 17:03:04 -0400 Subject: [PATCH 54/96] feat: richer default CLI color palette (#193) Smoke feedback on #112: bold-only headers and dim/yellow row tints read as nearly plain when mods are healthy. Extends the same mechanism (whole-row + last-column-only inline color, gated on TTY/NO_COLOR/--no-color, JSON never colored) with a fourth accent (cyan) and richer defaults: - Table headers are now bold+cyan (colorHeader), not bold-only - applies everywhere via printTable, so list/status/search/update all get it for free from one shared change. - lmm list -v: the common enabled+deployed row is now green-tinted (was untinted); yellow undeployed and dim disabled unchanged. - lmm search: an installed mod's whole row is green-tinted (was just the trailing [installed] marker). - lmm status: active profile and its "(active)" marker are green, mod/profile counts are cyan (including the games table's last column), Link Method is cyan, Last Deploy is green (dim when never deployed). - lmm mod show: Version fields are cyan, Update policy is colored per state (green for auto, yellow for pinned, plain for the notify default), and the mod name banner uses the same bold+cyan header accent. Every existing byte-stability test (no ANSI when color is disabled) still passes untouched; color-path tests updated for the new per-surface expectations, plus the alignment-preservation guard re-verified for each touched table. --- CHANGELOG.md | 2 +- cmd/lmm/color_test.go | 20 +++++++++- cmd/lmm/list.go | 12 +++--- cmd/lmm/list_color_test.go | 22 +++++------ cmd/lmm/mod.go | 14 +++++-- cmd/lmm/mod_show_color_test.go | 43 +++++++++++++++++++-- cmd/lmm/root.go | 32 ++++++++++++++-- cmd/lmm/search.go | 26 +++++++++---- cmd/lmm/search_color_test.go | 19 +++++++-- cmd/lmm/status.go | 38 ++++++++++++------ cmd/lmm/status_color_test.go | 70 ++++++++++++++++++++++++++++++++-- 11 files changed, 243 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba9ef1b..7d97d46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- CLI output is now colorized by default when stdout is a terminal, extending the existing `colorGreen`/`colorRed`/`colorYellow` accent mechanism (previously only used by `deploy`/`verify`) across `list`, `status`, `search`, `update`, `conflicts`, and `mod show`: table headers are bolded; a disabled mod's row in `lmm list -v` is dimmed and an enabled-but-undeployed row is accented yellow; `search`'s `[installed]` marker and `update`'s POLICY column color per row; `conflicts`' stale winner suffix, `mod show`'s pinned policy and lock line, and the `Enabled`/`Disabled` counts in `lmm status -g ` are accented; and the existing `✓`/`✗` success/failure markers extend to `update` and `mod`'s confirmation lines. Detection is TTY-aware (piped/redirected output stays plain) and layers on top of the existing `--no-color` flag and `NO_COLOR` env var, which continue to work unchanged; `--json` output is never colored. Table color is applied only to already-tabwriter-padded text (bolded headers, whole-row tints, or a table's genuinely last column) — never to interior cell values before they reach `text/tabwriter`, which pads columns by raw byte length and would misalign them (#112) +- CLI output is now colorized by default when stdout is a terminal, extending the existing `colorGreen`/`colorRed`/`colorYellow` accent mechanism (previously only used by `deploy`/`verify`) with a full 4-color palette (green/yellow/red/cyan, plus bold/dim) across `list`, `status`, `search`, `update`, `conflicts`, and `mod show`. Table headers are bold+cyan. `lmm list -v` tints the whole row: green for the common enabled+deployed case, yellow for enabled-but-undeployed, dim for disabled. `search` tints an installed mod's whole row green; `update`'s POLICY column colors per row. `status`/`mod show` color their values, not just the odd count: `lmm status -g `'s active profile and per-profile "(active)" marker are green, mod/profile counts are cyan, Link Method is cyan, Last Deploy is green (or dim when never deployed); `mod show`'s Version fields are cyan and its Update policy is colored per state (green for auto, yellow for pinned); `conflicts`' stale winner suffix is yellow; and the existing `✓`/`✗` success/failure markers extend to `update` and `mod`'s confirmation lines. Detection is TTY-aware (piped/redirected output stays plain) and layers on top of the existing `--no-color` flag and `NO_COLOR` env var (presence-only per no-color.org), which continue to work unchanged; `--json` output is never colored. Table color is applied only to already-tabwriter-padded text (accented headers, whole-row tints, or a table's genuinely last column) — never to interior cell values before they reach `text/tabwriter`, which pads columns by raw byte length and would misalign them (#112, #193) - **Icarus built-in mod source** (`internal/source/icarus`): a public, unauthenticated Firestore-backed catalog (Project Daedalus) — `lmm search`/`install`/`update` work against it like NexusMods/CurseForge. A `.exmodz` mod file now compiles into a deployable `_P.pak` at download time via a new, game-agnostic `internal/unrealpak` PAK reader/writer and the new `deploy_mode: compile` game setting; a plain `.pak` file from the same catalog is unaffected and deploys through the existing extract/copy pipeline unchanged. Base data tables are read directly from the installed game's own `data.pak`, so a compile always matches the installed game version and works entirely offline; `internal/unrealpak` reads both the stored and the Zlib-compressed entries that pak contains, using only the standard library (#136, #175) - `lmm game detect` now recognizes Icarus (Steam App ID `1149460`) and generates a complete `games.yaml` entry for it (`deploy_mode: compile`, `sources: {icarus: icarus}`) — no more hand-editing `games.yaml` to get started. The known-games schema (`steam-games.yaml`, built-in or your own override) gained two optional fields, `deploy_mode` and `sources`, generalizing detection beyond NexusMods-only games; every existing entry is unaffected (#177) - Custom `api` sources' `search` endpoint gains `{category}`/`{tags}` path placeholders, fed from `SearchQuery.Category`/`.Tags` (URL-escaped; multiple tags comma-joined) — previously these were silently dropped with no way for a declarative source to express category/tag filtering. A definition whose `search` path omits the new placeholders is unaffected: the values are computed but never substituted in, matching today's behavior exactly (#120) diff --git a/cmd/lmm/color_test.go b/cmd/lmm/color_test.go index 5855eb5..e5092f9 100644 --- a/cmd/lmm/color_test.go +++ b/cmd/lmm/color_test.go @@ -90,6 +90,8 @@ func TestColorHelpers_NoOpWhenColorDisabled(t *testing.T) { assert.Equal(t, "text", colorYellow("text")) assert.Equal(t, "text", colorBold("text")) assert.Equal(t, "text", colorDim("text")) + assert.Equal(t, "text", colorCyan("text")) + assert.Equal(t, "text", colorHeader("text")) } func TestColorHelpers_WrapWhenColorEnabled(t *testing.T) { @@ -101,6 +103,21 @@ func TestColorHelpers_WrapWhenColorEnabled(t *testing.T) { assert.Equal(t, ansiYellow+"text"+ansiReset, colorYellow("text")) assert.Equal(t, ansiBold+"text"+ansiReset, colorBold("text")) assert.Equal(t, ansiDim+"text"+ansiReset, colorDim("text")) + assert.Equal(t, ansiCyan+"text"+ansiReset, colorCyan("text")) +} + +// TestColorHeader_IsBoldAndCyan guards #193's richer header accent: bold +// alone read as nearly plain in smoke feedback, so a table/section header is +// now bold+cyan together, not bold-only. +func TestColorHeader_IsBoldAndCyan(t *testing.T) { + resetColorFlags(t) + withColorCapableStdout(t, true) + + got := colorHeader("text") + assert.Contains(t, got, ansiBold) + assert.Contains(t, got, ansiCyan) + assert.Contains(t, got, "text") + assert.Equal(t, "text", stripANSI(got)) } // TestPrintTable_ColorNeverShiftsColumnAlignment guards the exact regression @@ -143,7 +160,8 @@ func TestPrintTable_ColorNeverShiftsColumnAlignment(t *testing.T) { stripped := stripANSI(coloredOut.String()) assert.Equal(t, plainOut.String(), stripped, "color must not change the visible text or alignment") - assert.Contains(t, coloredOut.String(), ansiBold, "header line should be bolded when color is enabled") + assert.Contains(t, coloredOut.String(), ansiBold, "header line should be bold+cyan when color is enabled") + assert.Contains(t, coloredOut.String(), ansiCyan, "header line should be bold+cyan when color is enabled") assert.Contains(t, coloredOut.String(), ansiGreen) assert.Contains(t, coloredOut.String(), ansiRed) } diff --git a/cmd/lmm/list.go b/cmd/lmm/list.go index 3011d2f..e76a78e 100644 --- a/cmd/lmm/list.go +++ b/cmd/lmm/list.go @@ -198,10 +198,12 @@ func doList(cmd *cobra.Command, service *core.Service, game *domain.Game) error } // Row tinting only makes sense next to the columns it explains: ENABLED - // and DEPLOYED are verbose-only, so an anomaly (disabled, or enabled but - // not yet deployed) would be an unexplained color in the non-verbose - // table. Enabled+deployed - the common, unremarkable case - stays - // untinted (accent, not christmas tree). + // and DEPLOYED are verbose-only, so a tint would be unexplained color in + // the non-verbose table. #193: the common, healthy case (enabled+ + // deployed) gets a green tint too - #112's original "only flag + // anomalies" choice left the common case looking nearly plain in smoke + // feedback. Yellow (undeployed) and dim (disabled) still flag the + // anomalies. var rowColor func(int) func(string) string if verbose { rowColor = func(i int) func(string) string { @@ -214,7 +216,7 @@ func doList(cmd *cobra.Command, service *core.Service, game *domain.Game) error case !mods[i].Deployed: return colorYellow default: - return nil + return colorGreen } } } diff --git a/cmd/lmm/list_color_test.go b/cmd/lmm/list_color_test.go index 1ad4149..4a40481 100644 --- a/cmd/lmm/list_color_test.go +++ b/cmd/lmm/list_color_test.go @@ -59,17 +59,21 @@ func TestList_Verbose_PlainWhenColorDisabled(t *testing.T) { assert.Contains(t, out, "Enabled Undeployed") } +// TestList_Verbose_RowTinting guards #193's richer palette: the common, +// healthy case (enabled+deployed) must now be visibly colored too (green), +// not left untinted - #112's original "only flag anomalies" choice read as +// nearly plain in smoke feedback. Yellow (undeployed) and dim (disabled) +// are unchanged. func TestList_Verbose_RowTinting(t *testing.T) { tests := []struct { - name string - enabled bool - deployed bool - wantANSI string - wantNoOtherFor []string + name string + enabled bool + deployed bool + wantANSI string }{ {name: "disabled mod row is dimmed", enabled: false, deployed: false, wantANSI: ansiDim}, {name: "enabled but undeployed row is yellow", enabled: true, deployed: false, wantANSI: ansiYellow}, - {name: "enabled and deployed row is untinted", enabled: true, deployed: true, wantANSI: ""}, + {name: "enabled and deployed row is green", enabled: true, deployed: true, wantANSI: ansiGreen}, } for _, tt := range tests { @@ -85,11 +89,7 @@ func TestList_Verbose_RowTinting(t *testing.T) { row := rowFor(out, "Target Mod") require.NotEmpty(t, row) - if tt.wantANSI == "" { - assert.NotContains(t, row, "\x1b[", "an enabled+deployed row should not be tinted") - } else { - assert.Contains(t, row, tt.wantANSI) - } + assert.Contains(t, row, tt.wantANSI) }) } } diff --git a/cmd/lmm/mod.go b/cmd/lmm/mod.go index fc7a5f7..5ece4c6 100644 --- a/cmd/lmm/mod.go +++ b/cmd/lmm/mod.go @@ -630,9 +630,9 @@ func doModShow(ctx context.Context, svc *core.Service, game *domain.Game, modID // Human-readable output fmt.Printf("%s\n", strings.Repeat("=", 60)) - fmt.Printf("%s\n", colorBold(mod.Name)) + fmt.Printf("%s\n", colorHeader(mod.Name)) fmt.Printf("%s\n", strings.Repeat("=", 60)) - fmt.Printf("ID: %s Version: %s Author: %s\n", mod.ID, mod.Version, mod.Author) + fmt.Printf("ID: %s Version: %s Author: %s\n", mod.ID, colorCyan(mod.Version), mod.Author) if mod.Category != "" { fmt.Printf("Category: %s\n", mod.Category) } @@ -666,10 +666,16 @@ func doModShow(ctx context.Context, svc *core.Service, game *domain.Game, modID if installedInfo != nil { fmt.Println() - fmt.Printf("Installed: v%s (profile: %s)\n", installedInfo.Version, installedInfo.Profile) + fmt.Printf("Installed: v%s (profile: %s)\n", colorCyan(installedInfo.Version), installedInfo.Profile) policyDisplay := installedInfo.UpdatePolicy - if policyDisplay == "pinned" { + switch policyDisplay { + case "pinned": policyDisplay = colorYellow(policyDisplay) + case "auto": + // Auto is a positive, hands-off state - green, matching #193's + // "colored values, not just the odd count" (notify, the + // default, stays plain - there's nothing notable to flag). + policyDisplay = colorGreen(policyDisplay) } fmt.Printf(" Update policy: %s\n", policyDisplay) if installedInfo.Locked { diff --git a/cmd/lmm/mod_show_color_test.go b/cmd/lmm/mod_show_color_test.go index dd5e7b2..13ce481 100644 --- a/cmd/lmm/mod_show_color_test.go +++ b/cmd/lmm/mod_show_color_test.go @@ -26,8 +26,9 @@ func TestDoModShow_ColorPath_PlainByDefault(t *testing.T) { } // TestDoModShow_ColorPath_NameBolded_LockAccented: the mod's name banner is -// bolded, and a lock (a held-back/pending state) is accented yellow - -// matching the repo's established "pending"=yellow mapping. +// bold+cyan (#193 header richer accent, was bold-only in #112), and a lock +// (a held-back/pending state) is accented yellow - matching the repo's +// established "pending"=yellow mapping. func TestDoModShow_ColorPath_NameBolded_LockAccented(t *testing.T) { svc, game, src := setupDoModLockTest(t) seedLockableMod(t, svc, game, "a", "Mod A", "1.5") @@ -40,7 +41,7 @@ func TestDoModShow_ColorPath_NameBolded_LockAccented(t *testing.T) { return doModShow(context.Background(), svc, game, "a") }) - assert.Contains(t, out, colorBold("Mod A")) + assert.Contains(t, out, colorHeader("Mod A")) assert.Contains(t, out, colorYellow("locked at v1.2.3 — run 'lmm profile apply' to converge")) } @@ -60,3 +61,39 @@ func TestDoModShow_ColorPath_PinnedPolicyAccented(t *testing.T) { assert.Contains(t, out, "Update policy: "+colorYellow("pinned")) } + +// TestDoModShow_ColorPath_AutoPolicyAccented guards #193's richer palette: +// "auto" (a positive, hands-off state) is now green too, not just "pinned" - +// #112 only colored the odd-state-out. +func TestDoModShow_ColorPath_AutoPolicyAccented(t *testing.T) { + svc, game, src := setupDoModLockTest(t) + seedLockableMod(t, svc, game, "a", "Mod A", "1.5") + src.AddMod(&domain.Mod{ID: "a", SourceID: "src", GameID: game.ID, Name: "Mod A", Version: "1.5"}, nil) + require.NoError(t, svc.SetModUpdatePolicy("src", "a", game.ID, "default", domain.UpdateAuto)) + + resetColorFlags(t) + withColorCapableStdout(t, true) + out := captureStdout(t, func() error { + return doModShow(context.Background(), svc, game, "a") + }) + + assert.Contains(t, out, "Update policy: "+colorGreen("auto")) +} + +// TestDoModShow_ColorPath_VersionFieldsCyan guards #193's "key fields cyan" +// value accent: the header block's Version and the Installed line's version +// are both cyan. +func TestDoModShow_ColorPath_VersionFieldsCyan(t *testing.T) { + svc, game, src := setupDoModLockTest(t) + seedLockableMod(t, svc, game, "a", "Mod A", "1.5") + src.AddMod(&domain.Mod{ID: "a", SourceID: "src", GameID: game.ID, Name: "Mod A", Version: "1.5"}, nil) + + resetColorFlags(t) + withColorCapableStdout(t, true) + out := captureStdout(t, func() error { + return doModShow(context.Background(), svc, game, "a") + }) + + assert.Contains(t, out, "Version: "+colorCyan("1.5")) + assert.Contains(t, out, "Installed: v"+colorCyan("1.5")) +} diff --git a/cmd/lmm/root.go b/cmd/lmm/root.go index d623029..47f34c5 100644 --- a/cmd/lmm/root.go +++ b/cmd/lmm/root.go @@ -127,6 +127,7 @@ const ( ansiGreen = "\033[32m" ansiRed = "\033[31m" ansiYellow = "\033[33m" + ansiCyan = "\033[36m" ansiBold = "\033[1m" ansiDim = "\033[2m" ) @@ -173,10 +174,33 @@ func colorDim(s string) string { return ansiDim + s + ansiReset } +// colorCyan returns s with cyan ANSI when color is enabled, otherwise s. +// The palette's fourth accent, used for "key field" values (a version +// number, a link method, an active profile name) that deserve a visual +// anchor without implying good/bad the way green/yellow/red do. +func colorCyan(s string) string { + if !colorEnabled() { + return s + } + return ansiCyan + s + ansiReset +} + +// colorHeader returns s bold+cyan when color is enabled, otherwise s - the +// accent for a table header or section title. #193: bold alone read as +// nearly plain in smoke feedback, so headers now carry both bold and a +// color, not bold-only. +func colorHeader(s string) string { + if !colorEnabled() { + return s + } + return ansiBold + ansiCyan + s + ansiReset +} + // printTable writes a fully-flushed text/tabwriter table (buf) to os.Stdout, -// bolding the header line and applying rowColor's per-row wrapper (nil for -// no tint) when color is enabled. headerLines is the number of leading -// lines to skip when indexing data rows (2: header + dashed separator). +// accenting the header line (bold+cyan, via colorHeader) and applying +// rowColor's per-row wrapper (nil for no tint) when color is enabled. +// headerLines is the number of leading lines to skip when indexing data +// rows (2: header + dashed separator). // // Color is applied ONLY to buf's already-rendered, already-padded text - // never to a cell before it reaches the tabwriter. text/tabwriter computes @@ -200,7 +224,7 @@ func printTableTo(out io.Writer, buf *bytes.Buffer, headerLines int, rowColor fu } lines := strings.Split(text, "\n") if colorEnabled() { - lines[0] = colorBold(lines[0]) + lines[0] = colorHeader(lines[0]) if rowColor != nil { for i := headerLines; i < len(lines); i++ { if fn := rowColor(i - headerLines); fn != nil { diff --git a/cmd/lmm/search.go b/cmd/lmm/search.go index 595059b..09d414e 100644 --- a/cmd/lmm/search.go +++ b/cmd/lmm/search.go @@ -290,16 +290,20 @@ func doSearch(ctx context.Context, service *core.Service, game *domain.Game, arg return fmt.Errorf("writing separator: %w", err) } + // installedRows tracks each row's installed state in iteration order, so + // the whole row (not just the marker) can be green-tinted post-Flush - + // #193's richer palette (a cell-only accent read as too subtle in smoke + // feedback). Plain "[installed]" text is fed into the tabwriter; the + // row-level color wraps the already-padded line, matching printTable's + // "color only after Flush" contract. + var installedRows []bool for _, mod := range mods { installedMark := "" - if installedKeys[domain.ModKey(mod.SourceID, mod.ID)] { - // Safe to color inline here specifically because it's the LAST - // column: text/tabwriter never pads after the final cell, so - // this cell's byte length (inflated by ANSI codes) can't - // corrupt any other column's alignment. Do not do this for an - // interior column - see printTable's doc comment. - installedMark = colorGreen("[installed]") + installed := installedKeys[domain.ModKey(mod.SourceID, mod.ID)] + if installed { + installedMark = "[installed]" } + installedRows = append(installedRows, installed) if _, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", mod.ID, truncate(mod.Name, 40), @@ -314,7 +318,13 @@ func doSearch(ctx context.Context, service *core.Service, game *domain.Game, arg if err := w.Flush(); err != nil { return fmt.Errorf("flushing output: %w", err) } - if err := printTable(&buf, 2, nil); err != nil { + rowColor := func(i int) func(string) string { + if i >= 0 && i < len(installedRows) && installedRows[i] { + return colorGreen + } + return nil + } + if err := printTable(&buf, 2, rowColor); err != nil { return fmt.Errorf("writing table: %w", err) } diff --git a/cmd/lmm/search_color_test.go b/cmd/lmm/search_color_test.go index 5b4fd01..be8a4e3 100644 --- a/cmd/lmm/search_color_test.go +++ b/cmd/lmm/search_color_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "strings" "testing" "github.com/DonovanMods/linux-mod-manager/internal/core" @@ -89,7 +90,13 @@ func TestDoSearch_InstalledMarker_PlainWhenColorDisabled(t *testing.T) { assert.Contains(t, out, "[installed]") } -func TestDoSearch_InstalledMarker_GreenWhenTTY_AlignmentUnaffected(t *testing.T) { +// TestDoSearch_InstalledRow_GreenWhenTTY_AlignmentUnaffected guards #193's +// richer palette: an installed mod's ENTIRE row is now tinted green (whole- +// row, via printTable's rowColor), not just the trailing [installed] marker +// - #112's cell-only accent read as too subtle in smoke feedback. Still +// safe under the tabwriter alignment constraint since whole-row tinting +// wraps an already-flushed line, same as the header accent. +func TestDoSearch_InstalledRow_GreenWhenTTY_AlignmentUnaffected(t *testing.T) { svc, game := setupSearchColorTest(t) resetColorFlags(t) @@ -103,7 +110,13 @@ func TestDoSearch_InstalledMarker_GreenWhenTTY_AlignmentUnaffected(t *testing.T) return doSearch(context.Background(), svc, game, []string{"query"}) }) - assert.Contains(t, colored, ansiGreen+"[installed]"+ansiReset) - assert.Contains(t, colored, ansiBold, "header line should be bolded") + installedRow := rowFor(colored, "Installed Mod") + notInstalledRow := rowFor(colored, "Not Installed Mod") + require.NotEmpty(t, installedRow) + require.NotEmpty(t, notInstalledRow) + assert.True(t, strings.HasPrefix(installedRow, ansiGreen), "the whole installed row should be green-tinted") + assert.NotContains(t, notInstalledRow, "\x1b[", "a not-installed row should not be tinted") + assert.Contains(t, colored, ansiBold, "header line should be accented") + assert.Contains(t, colored, ansiCyan, "header line should be accented") assert.Equal(t, plain, stripANSI(colored), "color must not change the visible text or alignment") } diff --git a/cmd/lmm/status.go b/cmd/lmm/status.go index 9050868..b5662f2 100644 --- a/cmd/lmm/status.go +++ b/cmd/lmm/status.go @@ -113,28 +113,33 @@ func doStatus(service *core.Service) error { gameName += " (default)" } + // The last column (whichever count it is - MODS† in verbose, + // PROFILES otherwise) is safe to color inline: text/tabwriter never + // pads after the final cell, so this cell's inflated byte length + // can't misalign any column after it (see printTable's doc + // comment; do not do this for an interior column). if verbose { linkMethod := service.GetGameLinkMethod(game) linkStr := linkMethod.String() if game.LinkMethodExplicit { linkStr += "*" // Indicate per-game override } - if _, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d\t%d\n", + if _, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d\t%s\n", gameName, game.ID, truncate(game.InstallPath, 30), linkStr, len(profiles), - modCount, + colorCyan(strconv.Itoa(modCount)), ); err != nil { return fmt.Errorf("writing row: %w", err) } } else { - if _, err := fmt.Fprintf(w, "%s\t%s\t%d\t%d\n", + if _, err := fmt.Fprintf(w, "%s\t%s\t%d\t%s\n", gameName, truncate(game.InstallPath, 40), modCount, - len(profiles), + colorCyan(strconv.Itoa(len(profiles))), ); err != nil { return fmt.Errorf("writing row: %w", err) } @@ -323,11 +328,11 @@ func showGameStatus(service *core.Service, gameID string) error { activeProfile, activeErr := pm.GetDefault(gameID) switch { case activeErr == nil && activeProfile.LinkMethodExplicit: - fmt.Printf(" Link Method: %s (per-profile)\n", activeProfile.LinkMethod) + fmt.Printf(" Link Method: %s (per-profile)\n", colorCyan(activeProfile.LinkMethod.String())) case game.LinkMethodExplicit: - fmt.Printf(" Link Method: %s (per-game)\n", service.GetGameLinkMethod(game)) + fmt.Printf(" Link Method: %s (per-game)\n", colorCyan(service.GetGameLinkMethod(game).String())) case verbose: - fmt.Printf(" Link Method: %s (global default)\n", service.GetGameLinkMethod(game)) + fmt.Printf(" Link Method: %s (global default)\n", colorCyan(service.GetGameLinkMethod(game).String())) } // Show cache path @@ -363,17 +368,17 @@ func showGameStatus(service *core.Service, gameID string) error { for _, p := range profiles { defaultMark := "" if p.IsDefault { - defaultMark = " (active)" + defaultMark = colorGreen(" (active)") } - fmt.Printf(" - %s%s: %d mod(s)\n", p.Name, defaultMark, len(p.Mods)) + fmt.Printf(" - %s%s: %s mod(s)\n", p.Name, defaultMark, colorCyan(strconv.Itoa(len(p.Mods)))) } // Show installed mods count for active profile defaultProfile, err := pm.GetDefault(gameID) if err == nil { mods, _ := service.GetInstalledMods(gameID, defaultProfile.Name) - fmt.Printf("\nActive Profile: %s\n", defaultProfile.Name) - fmt.Printf(" Installed Mods: %d\n", len(mods)) + fmt.Printf("\nActive Profile: %s\n", colorGreen(defaultProfile.Name)) + fmt.Printf(" Installed Mods: %s\n", colorCyan(strconv.Itoa(len(mods)))) // Count enabled vs disabled var enabled, disabled int @@ -394,7 +399,16 @@ func showGameStatus(service *core.Service, gameID string) error { if err != nil { return fmt.Errorf("status: last deploy time: %w", err) } - fmt.Printf(" Last Deploy: %s\n", formatLastDeploy(lastDeploy)) + deployDisplay := formatLastDeploy(lastDeploy) + if lastDeploy == nil { + // "Never deployed" is a routine, expected state for a freshly + // added game - dimmed rather than red, same convention as a + // disabled mod (accent, not alarm). + deployDisplay = colorDim(deployDisplay) + } else { + deployDisplay = colorGreen(deployDisplay) + } + fmt.Printf(" Last Deploy: %s\n", deployDisplay) } return nil diff --git a/cmd/lmm/status_color_test.go b/cmd/lmm/status_color_test.go index f53a3a9..48e860e 100644 --- a/cmd/lmm/status_color_test.go +++ b/cmd/lmm/status_color_test.go @@ -41,8 +41,9 @@ func TestShowGameStatus_EnabledDisabledCounts_ColoredWhenTTY(t *testing.T) { } // TestDoStatus_TableHeader_BoldedWhenTTY_AlignmentUnaffected guards the -// "Configured Games:" summary table: header bolding must not perturb the -// tabwriter-computed column alignment (see printTable's doc comment). +// "Configured Games:" summary table: header accenting (#193: bold+cyan, not +// bold-only) must not perturb the tabwriter-computed column alignment (see +// printTable's doc comment). func TestDoStatus_TableHeader_BoldedWhenTTY_AlignmentUnaffected(t *testing.T) { svc, game := setupDoDeployTest(t) require.NoError(t, svc.AddGame(game)) @@ -59,6 +60,69 @@ func TestDoStatus_TableHeader_BoldedWhenTTY_AlignmentUnaffected(t *testing.T) { return doStatus(svc) }) - assert.Contains(t, colored, ansiBold, "table header should be bolded when color is enabled") + assert.Contains(t, colored, ansiBold, "table header should be accented when color is enabled") + assert.Contains(t, colored, ansiCyan, "table header should be accented when color is enabled") assert.Equal(t, plain, stripANSI(colored), "color must not change the visible text or alignment") } + +// TestDoStatus_LastColumnCount_CyanWhenTTY_AlignmentUnaffected guards #193's +// richer palette for the "Configured Games:" table: its last column (a mod +// or profile count, depending on --verbose) is safe to color inline since +// tabwriter never pads after the final cell (see printTable's doc comment). +func TestDoStatus_LastColumnCount_CyanWhenTTY_AlignmentUnaffected(t *testing.T) { + svc, game := setupDoDeployTest(t) + require.NoError(t, svc.AddGame(game)) + seedDeployableMod(t, svc, game, "1", "Test Mod", "a.esp") + + resetColorFlags(t) + withColorCapableStdout(t, false) + plain := captureStdout(t, func() error { + return doStatus(svc) + }) + + withColorCapableStdout(t, true) + colored := captureStdout(t, func() error { + return doStatus(svc) + }) + + assert.Contains(t, colored, ansiCyan+"1"+ansiReset, "the last column's count should be a cyan accent") + assert.Equal(t, plain, stripANSI(colored), "color must not change the visible text or alignment") +} + +// TestShowGameStatus_RicherValues_ColoredWhenTTY guards #193's expansion of +// showGameStatus beyond the Enabled/Disabled counts: the active profile +// name, installed-mod count, and per-profile mod count are now colored too +// - "colored values, not just the odd count". +func TestShowGameStatus_RicherValues_ColoredWhenTTY(t *testing.T) { + svc, game := setupDoDeployTest(t) + resetColorFlags(t) + require.NoError(t, svc.AddGame(game)) + seedDeployableMod(t, svc, game, "1", "Test Mod", "a.esp") + require.NoError(t, svc.NewProfileManager().SetDefault(game.ID, "default")) + + withColorCapableStdout(t, true) + out := captureStdout(t, func() error { + return showGameStatus(svc, game.ID) + }) + + assert.Contains(t, out, ansiGreen+"default"+ansiReset, "the active profile name should be accented green") + assert.Contains(t, out, ansiGreen+" (active)"+ansiReset, "the active profile marker should be accented green") + assert.Contains(t, out, ansiCyan+"1"+ansiReset, "a mod count should be a cyan accent") +} + +// TestShowGameStatus_RicherValues_PlainWhenColorDisabled is the byte- +// stability guard for the new value accents above. +func TestShowGameStatus_RicherValues_PlainWhenColorDisabled(t *testing.T) { + svc, game := setupDoDeployTest(t) + require.NoError(t, svc.AddGame(game)) + seedDeployableMod(t, svc, game, "1", "Test Mod", "a.esp") + require.NoError(t, svc.NewProfileManager().SetDefault(game.ID, "default")) + + out := captureStdout(t, func() error { + return showGameStatus(svc, game.ID) + }) + + assert.NotContains(t, out, "\x1b[") + assert.Contains(t, out, "Active Profile: default") + assert.Contains(t, out, " - default (active): 1 mod(s)") +} From 199f3ac94b9194caa3001e17bfd3ab8ba593f5d1 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 18:07:54 -0400 Subject: [PATCH 55/96] fix: unify list row-tint coloring across verbose and plain output (#193) lmm list and lmm list -v colored inconsistently: the row-state coloring (green enabled+deployed, yellow enabled-but-undeployed, dim disabled) only ever ran on the verbose branch, since it lived inline inside `if verbose { ... }`. Extracted the state->color decision into a shared modRowColor(enabled, deployed bool) helper (root.go, next to the other color primitives) and wired doList to call it unconditionally - a mod's health now renders identically whether or not -v is passed, since the color reflects the mod's actual state, not which columns happen to be displayed. Swept every other tabwriter-based table in cmd/lmm for the same inconsistency: none of them display per-mod enabled/deployed state (profile list/reorder show profile names or load order; status/ search/update were already unified in #193's first pass), so list.go was the only affected table. Amended the CHANGELOG's #112 entry, which had said "lmm list -v tints the whole row" - now describes the shared, flag-independent behavior. --- CHANGELOG.md | 2 +- cmd/lmm/list.go | 33 ++++++--------- cmd/lmm/list_color_test.go | 83 ++++++++++++++++++++++++++++++++++++++ cmd/lmm/root.go | 23 +++++++++++ 4 files changed, 119 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d97d46..2c5babd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- CLI output is now colorized by default when stdout is a terminal, extending the existing `colorGreen`/`colorRed`/`colorYellow` accent mechanism (previously only used by `deploy`/`verify`) with a full 4-color palette (green/yellow/red/cyan, plus bold/dim) across `list`, `status`, `search`, `update`, `conflicts`, and `mod show`. Table headers are bold+cyan. `lmm list -v` tints the whole row: green for the common enabled+deployed case, yellow for enabled-but-undeployed, dim for disabled. `search` tints an installed mod's whole row green; `update`'s POLICY column colors per row. `status`/`mod show` color their values, not just the odd count: `lmm status -g `'s active profile and per-profile "(active)" marker are green, mod/profile counts are cyan, Link Method is cyan, Last Deploy is green (or dim when never deployed); `mod show`'s Version fields are cyan and its Update policy is colored per state (green for auto, yellow for pinned); `conflicts`' stale winner suffix is yellow; and the existing `✓`/`✗` success/failure markers extend to `update` and `mod`'s confirmation lines. Detection is TTY-aware (piped/redirected output stays plain) and layers on top of the existing `--no-color` flag and `NO_COLOR` env var (presence-only per no-color.org), which continue to work unchanged; `--json` output is never colored. Table color is applied only to already-tabwriter-padded text (accented headers, whole-row tints, or a table's genuinely last column) — never to interior cell values before they reach `text/tabwriter`, which pads columns by raw byte length and would misalign them (#112, #193) +- CLI output is now colorized by default when stdout is a terminal, extending the existing `colorGreen`/`colorRed`/`colorYellow` accent mechanism (previously only used by `deploy`/`verify`) with a full 4-color palette (green/yellow/red/cyan, plus bold/dim) across `list`, `status`, `search`, `update`, `conflicts`, and `mod show`. Table headers are bold+cyan. `lmm list` tints the whole row identically with or without `-v` (the row-tint decision is a single shared helper keyed on the mod's actual state, not the display flag): green for the common enabled+deployed case, yellow for enabled-but-undeployed, dim for disabled. `search` tints an installed mod's whole row green; `update`'s POLICY column colors per row. `status`/`mod show` color their values, not just the odd count: `lmm status -g `'s active profile and per-profile "(active)" marker are green, mod/profile counts are cyan, Link Method is cyan, Last Deploy is green (or dim when never deployed); `mod show`'s Version fields are cyan and its Update policy is colored per state (green for auto, yellow for pinned); `conflicts`' stale winner suffix is yellow; and the existing `✓`/`✗` success/failure markers extend to `update` and `mod`'s confirmation lines. Detection is TTY-aware (piped/redirected output stays plain) and layers on top of the existing `--no-color` flag and `NO_COLOR` env var (presence-only per no-color.org), which continue to work unchanged; `--json` output is never colored. Table color is applied only to already-tabwriter-padded text (accented headers, whole-row tints, or a table's genuinely last column) — never to interior cell values before they reach `text/tabwriter`, which pads columns by raw byte length and would misalign them (#112, #193) - **Icarus built-in mod source** (`internal/source/icarus`): a public, unauthenticated Firestore-backed catalog (Project Daedalus) — `lmm search`/`install`/`update` work against it like NexusMods/CurseForge. A `.exmodz` mod file now compiles into a deployable `_P.pak` at download time via a new, game-agnostic `internal/unrealpak` PAK reader/writer and the new `deploy_mode: compile` game setting; a plain `.pak` file from the same catalog is unaffected and deploys through the existing extract/copy pipeline unchanged. Base data tables are read directly from the installed game's own `data.pak`, so a compile always matches the installed game version and works entirely offline; `internal/unrealpak` reads both the stored and the Zlib-compressed entries that pak contains, using only the standard library (#136, #175) - `lmm game detect` now recognizes Icarus (Steam App ID `1149460`) and generates a complete `games.yaml` entry for it (`deploy_mode: compile`, `sources: {icarus: icarus}`) — no more hand-editing `games.yaml` to get started. The known-games schema (`steam-games.yaml`, built-in or your own override) gained two optional fields, `deploy_mode` and `sources`, generalizing detection beyond NexusMods-only games; every existing entry is unaffected (#177) - Custom `api` sources' `search` endpoint gains `{category}`/`{tags}` path placeholders, fed from `SearchQuery.Category`/`.Tags` (URL-escaped; multiple tags comma-joined) — previously these were silently dropped with no way for a declarative source to express category/tag filtering. A definition whose `search` path omits the new placeholders is unaffected: the values are computed but never substituted in, matching today's behavior exactly (#120) diff --git a/cmd/lmm/list.go b/cmd/lmm/list.go index e76a78e..703d981 100644 --- a/cmd/lmm/list.go +++ b/cmd/lmm/list.go @@ -197,28 +197,19 @@ func doList(cmd *cobra.Command, service *core.Service, game *domain.Game) error return fmt.Errorf("flushing output: %w", err) } - // Row tinting only makes sense next to the columns it explains: ENABLED - // and DEPLOYED are verbose-only, so a tint would be unexplained color in - // the non-verbose table. #193: the common, healthy case (enabled+ - // deployed) gets a green tint too - #112's original "only flag - // anomalies" choice left the common case looking nearly plain in smoke - // feedback. Yellow (undeployed) and dim (disabled) still flag the - // anomalies. - var rowColor func(int) func(string) string - if verbose { - rowColor = func(i int) func(string) string { - if i < 0 || i >= len(mods) { - return nil - } - switch { - case !mods[i].Enabled: - return colorDim - case !mods[i].Deployed: - return colorYellow - default: - return colorGreen - } + // Row tinting reflects each mod's actual enabled/deployed state + // regardless of --verbose: the non-verbose table doesn't SHOW the + // ENABLED/DEPLOYED columns, but the mod's health is exactly as real + // there as in the verbose table, and a user comparing `lmm list` against + // `lmm list -v` should see the identical color for the identical mod + // (#193 round 2 - the row-tint decision had only ever been wired up on + // the verbose branch, so plain `lmm list` stayed uncolored while `-v` + // wasn't). + rowColor := func(i int) func(string) string { + if i < 0 || i >= len(mods) { + return nil } + return modRowColor(mods[i].Enabled, mods[i].Deployed) } if err := printTable(&buf, 2, rowColor); err != nil { return fmt.Errorf("writing table: %w", err) diff --git a/cmd/lmm/list_color_test.go b/cmd/lmm/list_color_test.go index 4a40481..fe46464 100644 --- a/cmd/lmm/list_color_test.go +++ b/cmd/lmm/list_color_test.go @@ -6,10 +6,26 @@ import ( "github.com/DonovanMods/linux-mod-manager/internal/core" "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +// listNonVerbose runs doList without --verbose and returns stdout - the +// non-verbose counterpart to pin_visibility_test.go's listVerbose, for +// tests guarding that row tinting is identical on both paths (#193 round 2: +// it had only landed on the verbose branch). +func listNonVerbose(t *testing.T, svc *core.Service, game *domain.Game) string { + t.Helper() + oldVerbose, oldJSON := verbose, jsonOutput + verbose, jsonOutput = false, false + t.Cleanup(func() { verbose, jsonOutput = oldVerbose, oldJSON }) + + return captureStdout(t, func() error { + return doList(&cobra.Command{}, svc, game) + }) +} + // seedModWithState installs modID/name with an explicit enabled/deployed // combination (seedDeployableMod always seeds Enabled: true, Deployed: // false, which isn't enough to exercise every row-tint branch). @@ -113,3 +129,70 @@ func TestList_Verbose_ColorNeverBreaksAlignment(t *testing.T) { assert.Equal(t, plain, stripANSI(colored)) } + +// TestList_NonVerbose_PlainWhenColorDisabled mirrors +// TestList_Verbose_PlainWhenColorDisabled for the non-verbose path. +func TestList_NonVerbose_PlainWhenColorDisabled(t *testing.T) { + svc, game := setupDoDeployTest(t) + seedModWithState(t, svc, game, "a", "Enabled Deployed", true, true) + seedModWithState(t, svc, game, "b", "Disabled Mod", false, false) + seedModWithState(t, svc, game, "c", "Enabled Undeployed", true, false) + + out := listNonVerbose(t, svc, game) + + assert.NotContains(t, out, "\x1b[", "plain output must never contain ANSI escapes") + assert.Contains(t, out, "Disabled Mod") + assert.Contains(t, out, "Enabled Undeployed") +} + +// TestList_NonVerbose_RowTinting is the round-2 regression guard: smoke +// feedback found `lmm list` (no -v) and `lmm list -v` colored +// INCONSISTENTLY, because the row-tint decision only ever ran on the +// verbose branch. Plain `lmm list` doesn't show the ENABLED/DEPLOYED +// columns, but the mods themselves are still enabled/deployed or not, so +// the same row-tint rules must apply identically here. +func TestList_NonVerbose_RowTinting(t *testing.T) { + tests := []struct { + name string + enabled bool + deployed bool + wantANSI string + }{ + {name: "disabled mod row is dimmed", enabled: false, deployed: false, wantANSI: ansiDim}, + {name: "enabled but undeployed row is yellow", enabled: true, deployed: false, wantANSI: ansiYellow}, + {name: "enabled and deployed row is green", enabled: true, deployed: true, wantANSI: ansiGreen}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + svc, game := setupDoDeployTest(t) + resetColorFlags(t) + seedModWithState(t, svc, game, "x", "Target Mod", tt.enabled, tt.deployed) + + withColorCapableStdout(t, true) + out := listNonVerbose(t, svc, game) + + row := rowFor(out, "Target Mod") + require.NotEmpty(t, row) + assert.Contains(t, row, tt.wantANSI) + }) + } +} + +// TestList_NonVerbose_ColorNeverBreaksAlignment mirrors +// TestList_Verbose_ColorNeverBreaksAlignment for the non-verbose table. +func TestList_NonVerbose_ColorNeverBreaksAlignment(t *testing.T) { + svc, game := setupDoDeployTest(t) + resetColorFlags(t) + seedModWithState(t, svc, game, "a", "Enabled Deployed", true, true) + seedModWithState(t, svc, game, "b", "Disabled Mod", false, false) + seedModWithState(t, svc, game, "c", "Enabled Undeployed", true, false) + + withColorCapableStdout(t, false) + plain := listNonVerbose(t, svc, game) + + withColorCapableStdout(t, true) + colored := listNonVerbose(t, svc, game) + + assert.Equal(t, plain, stripANSI(colored)) +} diff --git a/cmd/lmm/root.go b/cmd/lmm/root.go index 47f34c5..eaae6e3 100644 --- a/cmd/lmm/root.go +++ b/cmd/lmm/root.go @@ -196,6 +196,29 @@ func colorHeader(s string) string { return ansiBold + ansiCyan + s + ansiReset } +// modRowColor returns the row-tint color function for a mod's +// enabled/deployed state: dim for disabled (a routine, expected state - not +// an error), yellow for enabled-but-not-yet-deployed (drift worth noticing), +// green for enabled+deployed (the common, healthy case - #193: originally +// left untinted, which read as nearly plain in smoke feedback). +// +// The single shared decision for any command that lists mod rows keyed on +// this state - do not reimplement this switch inline in a second call site; +// a mod's health must render identically everywhere it's shown, independent +// of which columns that particular view happens to display (#193 round 2: +// list -v and plain list colored inconsistently because the decision only +// existed inline in the verbose branch). +func modRowColor(enabled, deployed bool) func(string) string { + switch { + case !enabled: + return colorDim + case !deployed: + return colorYellow + default: + return colorGreen + } +} + // printTable writes a fully-flushed text/tabwriter table (buf) to os.Stdout, // accenting the header line (bold+cyan, via colorHeader) and applying // rowColor's per-row wrapper (nil for no tint) when color is enabled. From a3afd3ab5c601ee8420dd3d84e520b54dd69bb27 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 18:14:06 -0400 Subject: [PATCH 56/96] test: prove TTY gate, not --no-color, keeps piped list plain (#193 review) Four "plain when color disabled" tests called setupDoDeployTest, which forces noColor=true, then asserted no ANSI escapes appear - without resetting that flag, the assertion only proved --no-color suppresses color, not that colorEnabled()'s TTY-detection gate itself keeps piped/non-TTY output plain. A regression that dropped the stdoutColorCapable() check entirely (defaulting color to on) would have slipped through undetected, since noColor=true masked it either way. Added resetColorFlags(t) to each (list_color_test.go's verbose and non-verbose plainness tests, flagged by Copilot review on PR #195; plus two siblings with the identical pattern found sweeping the other color tests, status_color_test.go's Enabled/Disabled-counts and richer-values plainness tests) so they rely on captureStdout's real os.Pipe swap - a genuine non-TTY stdout - rather than the --no-color flag. Verified by temporarily bypassing the TTY check in colorEnabled() and confirming all four now fail as expected; reverted before committing. --- cmd/lmm/list_color_test.go | 12 +++++++++++- cmd/lmm/status_color_test.go | 10 ++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/cmd/lmm/list_color_test.go b/cmd/lmm/list_color_test.go index fe46464..9a39e82 100644 --- a/cmd/lmm/list_color_test.go +++ b/cmd/lmm/list_color_test.go @@ -61,9 +61,13 @@ func rowFor(out, name string) string { // guard: with color off (the default for piped/non-TTY output, and every // test that doesn't force stdoutColorCapable), `lmm list -v` output must // carry no ANSI escapes at all, regardless of each mod's enabled/deployed -// state. +// state. resetColorFlags undoes setupDoDeployTest's noColor=true so this +// proves the real TTY-detection gate keeps piped output plain, not the +// --no-color flag - captureStdout's os.Pipe swap gives stdoutColorCapable() +// a genuine non-TTY answer without any test-seam override. func TestList_Verbose_PlainWhenColorDisabled(t *testing.T) { svc, game := setupDoDeployTest(t) + resetColorFlags(t) seedModWithState(t, svc, game, "a", "Enabled Deployed", true, true) seedModWithState(t, svc, game, "b", "Disabled Mod", false, false) seedModWithState(t, svc, game, "c", "Enabled Undeployed", true, false) @@ -132,8 +136,14 @@ func TestList_Verbose_ColorNeverBreaksAlignment(t *testing.T) { // TestList_NonVerbose_PlainWhenColorDisabled mirrors // TestList_Verbose_PlainWhenColorDisabled for the non-verbose path. +// resetColorFlags undoes setupDoDeployTest's noColor=true (review finding on +// PR #195: without it, this only proved --no-color suppresses color, not +// that the TTY-detection gate itself keeps piped output plain - a color +// leak into non-TTY output with noColor=false would have slipped through +// undetected). func TestList_NonVerbose_PlainWhenColorDisabled(t *testing.T) { svc, game := setupDoDeployTest(t) + resetColorFlags(t) seedModWithState(t, svc, game, "a", "Enabled Deployed", true, true) seedModWithState(t, svc, game, "b", "Disabled Mod", false, false) seedModWithState(t, svc, game, "c", "Enabled Undeployed", true, false) diff --git a/cmd/lmm/status_color_test.go b/cmd/lmm/status_color_test.go index 48e860e..ab602b1 100644 --- a/cmd/lmm/status_color_test.go +++ b/cmd/lmm/status_color_test.go @@ -9,9 +9,12 @@ import ( // TestShowGameStatus_EnabledDisabledCounts_PlainWhenColorDisabled is the // byte-stability guard: with color off (the default), the Enabled/Disabled -// summary line must carry no ANSI escapes. +// summary line must carry no ANSI escapes. resetColorFlags undoes +// setupDoDeployTest's noColor=true so this proves the real TTY-detection +// gate keeps piped output plain, not the --no-color flag. func TestShowGameStatus_EnabledDisabledCounts_PlainWhenColorDisabled(t *testing.T) { svc, game := setupDoDeployTest(t) + resetColorFlags(t) require.NoError(t, svc.AddGame(game)) seedDeployableMod(t, svc, game, "1", "Enabled Mod", "a.esp") seedModWithState(t, svc, game, "2", "Disabled Mod", false, false) @@ -111,9 +114,12 @@ func TestShowGameStatus_RicherValues_ColoredWhenTTY(t *testing.T) { } // TestShowGameStatus_RicherValues_PlainWhenColorDisabled is the byte- -// stability guard for the new value accents above. +// stability guard for the new value accents above. resetColorFlags undoes +// setupDoDeployTest's noColor=true so this proves the real TTY-detection +// gate keeps piped output plain, not the --no-color flag. func TestShowGameStatus_RicherValues_PlainWhenColorDisabled(t *testing.T) { svc, game := setupDoDeployTest(t) + resetColorFlags(t) require.NoError(t, svc.AddGame(game)) seedDeployableMod(t, svc, game, "1", "Test Mod", "a.esp") require.NoError(t, svc.NewProfileManager().SetDefault(game.ID, "default")) From 8aeb952f269d5e03d4e2694858470dd864d8c709 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 18:40:30 -0400 Subject: [PATCH 57/96] feat: record base-pak fingerprint at compile time, detect staleness (#196) Adds the marker/cache-layer machinery for base-staleness detection (the "Friday problem" - a weekly Icarus data.pak refresh silently reverting a compiled mod's patched tables, with nothing to notice): - unrealpak.Reader.IndexHash(): cheap, footer-only fingerprint of a pak's content, never reads file payloads. - cache: MarkBaseIndexHash/BaseIndexHashes (reserved per-file marker, mirrors the FileManifests marker system) and RetainedSourceName (reserved naming for a compiled file's retained .exmodz source) - both excluded from ListFiles/Size/deploy like every other .lmm-* entry. - Both compile sites (download's DownloadModToCache, import's Importer.Import) now stage the base pak's IndexHash and a retained copy of the original .exmodz into the SAME atomic commit as the compiled pak, via a new stageCompileFingerprint helper. - Service.CheckBaseStaleness / CheckGameUpdates: local, offline staleness scan comparing each compiled entry's recorded fingerprint against the live base pak, merged with Updater.CheckUpdates as the single seam CLI/TUI will check updates through. Per review: only entries carrying a fingerprint participate - a missing one is skipped, not flagged stale, since a DeployCompile game's catalog can also serve never-compiled prebuilt .pak files indistinguishable from a pre-#196 compile by any local signal alone. domain.Update gains RecompileNeeded. CLI/TUI wiring and update-apply (recompile in place, lock/pin semantics, verify warning) land in a follow-up commit on this branch. --- cmd/lmm/install_compile_test.go | 14 +- internal/core/importer.go | 11 ++ internal/core/service.go | 42 ++++ internal/core/service_base_staleness_test.go | 180 ++++++++++++++++++ .../core/service_compile_fingerprint_test.go | 129 +++++++++++++ internal/core/service_icarus_compile_test.go | 18 +- internal/core/service_import_compile_test.go | 8 +- internal/core/updater.go | 94 +++++++++ internal/domain/mod.go | 8 + internal/storage/cache/cache.go | 78 ++++++++ internal/storage/cache/cache_test.go | 84 ++++++++ internal/unrealpak/reader.go | 13 +- internal/unrealpak/reader_test.go | 51 +++++ 13 files changed, 722 insertions(+), 8 deletions(-) create mode 100644 internal/core/service_base_staleness_test.go create mode 100644 internal/core/service_compile_fingerprint_test.go diff --git a/cmd/lmm/install_compile_test.go b/cmd/lmm/install_compile_test.go index ffd18f6..1c53202 100644 --- a/cmd/lmm/install_compile_test.go +++ b/cmd/lmm/install_compile_test.go @@ -7,10 +7,22 @@ import ( "testing" "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +// writeFakeBasePak writes a real, minimal but VALID pak at path (#196: the +// compile branch now opens the base pak itself to read its footer IndexHash +// as a compile fingerprint, so a bare byte-stub file no longer parses). +func writeFakeBasePak(t *testing.T, path string) { + t.Helper() + w, err := unrealpak.Create(path) + require.NoError(t, err) + require.NoError(t, w.AddFile("Data/D_Fixture.json", []byte(`{"fixture":true}`))) + require.NoError(t, w.Close()) +} + // compilerInstallSource wraps fakeInstallSource with a source.Compiler // implementation, so `lmm install` can drive a real DeployCompile game // end-to-end through the CLI's exact console-output path (mirrors @@ -46,7 +58,7 @@ func TestDoInstall_DeployCompile_AnnouncesCompiling(t *testing.T) { basePak := filepath.Join(game.InstallPath, "Icarus", "Content", "Data", "data.pak") require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) - require.NoError(t, os.WriteFile(basePak, []byte("fake-base-pak"), 0o644)) + writeFakeBasePak(t, basePak) compiler := &compilerInstallSource{fakeInstallSource: src} // Re-register under the same ID so doInstall's resolved source is the diff --git a/internal/core/importer.go b/internal/core/importer.go index 17075bb..392a6ca 100644 --- a/internal/core/importer.go +++ b/internal/core/importer.go @@ -169,6 +169,17 @@ func (i *Importer) Import(ctx context.Context, archivePath string, game *domain. if err := copyFileStreaming(compiledPath, filepath.Join(stagePath, destName)); err != nil { return nil, fmt.Errorf("staging compiled mod: %w", err) } + // #196: stage the compile fingerprint (base pak IndexHash) and + // retained source keyed by destName - Import has no real source + // file ID the way a download does (DownloadableFile.ID is resolved + // later, outside Import, only when --id was given), so the compiled + // output's own filename is the stable per-entry key instead. A + // re-import of the same archive name replaces this entry outright + // (prepareUnseededStaging), so destName never collides across + // generations of the same mod. + if err := stageCompileFingerprint(stagePath, destName, basePakPath, archivePath); err != nil { + return nil, err + } if err := commitStagedCache(cachePath, stagePath); err != nil { return nil, err } diff --git a/internal/core/service.go b/internal/core/service.go index 7979c78..728f662 100644 --- a/internal/core/service.go +++ b/internal/core/service.go @@ -21,6 +21,7 @@ import ( "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" "github.com/DonovanMods/linux-mod-manager/internal/storage/config" "github.com/DonovanMods/linux-mod-manager/internal/storage/db" + "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" "golang.org/x/sync/errgroup" ) @@ -552,6 +553,9 @@ func (s *Service) DownloadModToCache(ctx context.Context, gameCache *cache.Cache if err := compiler.Compile(ctx, basePakPath, archivePath, destPath); err != nil { return nil, fmt.Errorf("compiling mod: %w", err) } + if err := stageCompileFingerprint(stagePath, file.ID, basePakPath, archivePath); err != nil { + return nil, err + } if err := commitStagedCacheWithMarker(cachePath, stagePath, file.ID, []string{destName}); err != nil { return nil, err } @@ -992,6 +996,44 @@ func compiledFileName(sourceFileName string) string { return base + "_P.pak" } +// basePakIndexHash opens basePakPath and returns its footer IndexHash +// (#196) - cheap (footer + primary-index region only; unrealpak.Open never +// reads a pak's actual file payloads), matching the base pak Compile itself +// already opens to read patched tables from, so this adds no new I/O +// pattern to the compile path. +func basePakIndexHash(basePakPath string) (string, error) { + r, err := unrealpak.Open(basePakPath) + if err != nil { + return "", fmt.Errorf("reading base pak for compile fingerprint: %w", err) + } + defer r.Close() //nolint:errcheck + return r.IndexHash(), nil +} + +// stageCompileFingerprint stages fileID's #196 compile fingerprint into +// stagePath: the base pak's IndexHash (cache.MarkBaseIndexHash) and a copy +// of the original .exmodz (cache.RetainedSourceName), so a later staleness +// check can detect the base pak changing, and a later recompile can run +// offline. Both land in the SAME atomic commit as the compiled pak - see +// commitStagedCache/commitStagedCacheWithMarker - so a partial write here +// can never separate a compiled pak from its fingerprint or retained +// source. Called by both compile sites (download: DownloadModToCache; +// import: Importer.Import) after Compile succeeds, before the commit. +func stageCompileFingerprint(stagePath, fileID, basePakPath, sourceFilePath string) error { + indexHash, err := basePakIndexHash(basePakPath) + if err != nil { + return err + } + if err := cache.MarkBaseIndexHash(stagePath, fileID, indexHash); err != nil { + return err + } + retainedPath := filepath.Join(stagePath, cache.RetainedSourceName(fileID)) + if err := copyFileStreaming(sourceFilePath, retainedPath); err != nil { + return fmt.Errorf("retaining compile source: %w", err) + } + return nil +} + // GetGame retrieves a game by ID func (s *Service) GetGame(gameID string) (*domain.Game, error) { game, ok := s.games[gameID] diff --git a/internal/core/service_base_staleness_test.go b/internal/core/service_base_staleness_test.go new file mode 100644 index 0000000..a7670ad --- /dev/null +++ b/internal/core/service_base_staleness_test.go @@ -0,0 +1,180 @@ +package core_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/core" + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" + "github.com/stretchr/testify/require" +) + +// newStalenessTestService builds a DeployCompile game with a real (fixture) +// base pak installed at installDir, and a service whose cache/config live +// under fresh temp dirs. Returns the service, the game, and the base pak's +// path so tests can rewrite it to simulate a base-pak refresh. +func newStalenessTestService(t *testing.T) (*core.Service, *domain.Game, string) { + t.Helper() + + installDir := t.TempDir() + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + writeFakeBasePak(t, basePak) + + cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} + svc, err := core.NewService(cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + game := &domain.Game{ID: "icarus", InstallPath: installDir, ModPath: t.TempDir(), DeployMode: domain.DeployCompile} + require.NoError(t, svc.AddGame(game)) + + return svc, game, basePak +} + +// seedCompiledMod stages a fake compiled entry directly through the cache +// (bypassing Compile/Importer entirely - CheckBaseStaleness only reads +// markers, so this is a faster, more direct way to set up its inputs than +// driving a full compile), recording fingerprint as the file's base-index +// hash if non-empty (empty simulates a pre-#196 entry with NO marker at +// all). +func seedCompiledMod(t *testing.T, svc *core.Service, game *domain.Game, mod domain.InstalledMod, fingerprint string) { + t.Helper() + gameCache := svc.GetGameCache(game) + versionDir := gameCache.ModPath(game.ID, mod.SourceID, mod.ID, mod.Version) + require.NoError(t, os.MkdirAll(versionDir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(versionDir, "Fake_P.pak"), []byte("compiled"), 0o644)) + if fingerprint != "" { + require.NoError(t, cache.MarkBaseIndexHash(versionDir, "fake-file-id", fingerprint)) + } +} + +func TestCheckBaseStaleness_FingerprintMatch_NotStale(t *testing.T) { + svc, game, basePak := newStalenessTestService(t) + liveHash := basePakIndexHash(t, basePak) + + mod := domain.InstalledMod{Mod: domain.Mod{ID: "bear-mount", SourceID: "fake-compiler", Version: "1.0"}} + seedCompiledMod(t, svc, game, mod, liveHash) + + stale, err := svc.CheckBaseStaleness(game, []domain.InstalledMod{mod}) + require.NoError(t, err) + require.Empty(t, stale, "a fingerprint matching the live base pak must not be reported stale") +} + +func TestCheckBaseStaleness_FingerprintMismatch_Stale(t *testing.T) { + svc, game, _ := newStalenessTestService(t) + + mod := domain.InstalledMod{Mod: domain.Mod{ID: "bear-mount", SourceID: "fake-compiler", Version: "1.0"}} + seedCompiledMod(t, svc, game, mod, "0000000000000000000000000000000000dead") // deliberately wrong + + stale, err := svc.CheckBaseStaleness(game, []domain.InstalledMod{mod}) + require.NoError(t, err) + require.Len(t, stale, 1) + require.True(t, stale[0].RecompileNeeded) + require.Equal(t, mod.Version, stale[0].NewVersion, "NewVersion must equal the current version - the mod hasn't changed, only the base pak has") + require.Equal(t, mod.ID, stale[0].InstalledMod.ID) +} + +// TestCheckBaseStaleness_MissingFingerprint_NotStale pins the #196-review +// amendment: a compiled entry with NO base-index marker (predates #196, or +// is actually a never-compiled prebuilt .pak - the two are locally +// indistinguishable) is skipped, not flagged. Flagging it would false- +// positive forever on plain prebuilt .pak mods, which a DeployCompile +// game's catalog can also legitimately serve (isExmodzFile only routes +// .exmodz through Compile). +func TestCheckBaseStaleness_MissingFingerprint_NotStale(t *testing.T) { + svc, game, _ := newStalenessTestService(t) + + mod := domain.InstalledMod{Mod: domain.Mod{ID: "bear-mount", SourceID: "fake-compiler", Version: "1.0"}} + seedCompiledMod(t, svc, game, mod, "") // no marker at all + + stale, err := svc.CheckBaseStaleness(game, []domain.InstalledMod{mod}) + require.NoError(t, err) + require.Empty(t, stale, "a missing fingerprint must be skipped, not flagged stale") +} + +// TestCheckBaseStaleness_PinnedModIncluded pins design point 3: pinning +// fixes the mod VERSION, not the base pak, so a pinned mod's staleness must +// still be reported (unlike Updater.CheckUpdates, which filters pinned mods +// out entirely via UpdateCheckable). +func TestCheckBaseStaleness_PinnedModIncluded(t *testing.T) { + svc, game, _ := newStalenessTestService(t) + + mod := domain.InstalledMod{Mod: domain.Mod{ID: "bear-mount", SourceID: "fake-compiler", Version: "1.0"}, UpdatePolicy: domain.UpdatePinned} + seedCompiledMod(t, svc, game, mod, "0000000000000000000000000000000000dead") + + stale, err := svc.CheckBaseStaleness(game, []domain.InstalledMod{mod}) + require.NoError(t, err) + require.Len(t, stale, 1, "a pinned mod must still be checked for base staleness") +} + +// TestCheckBaseStaleness_LocalModIncluded: a pure local import (SourceID == +// domain.SourceLocal) has no remote to check, but it CAN go stale against +// the base pak - this check is entirely local/offline, so it must not skip +// local mods the way Updater.CheckUpdates does. +func TestCheckBaseStaleness_LocalModIncluded(t *testing.T) { + svc, game, _ := newStalenessTestService(t) + + mod := domain.InstalledMod{Mod: domain.Mod{ID: "bear-mount", SourceID: domain.SourceLocal, Version: "1.0"}} + seedCompiledMod(t, svc, game, mod, "0000000000000000000000000000000000dead") + + stale, err := svc.CheckBaseStaleness(game, []domain.InstalledMod{mod}) + require.NoError(t, err) + require.Len(t, stale, 1) +} + +// TestCheckBaseStaleness_NonCompileGame_NoOp: a DeployExtract/DeployCopy +// game has no base pak concept at all - CheckBaseStaleness must be an +// unconditional no-op rather than erroring on a missing base pak path. +func TestCheckBaseStaleness_NonCompileGame_NoOp(t *testing.T) { + cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} + svc, err := core.NewService(cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + game := &domain.Game{ID: "skyrim-se", ModPath: t.TempDir(), DeployMode: domain.DeployExtract} + require.NoError(t, svc.AddGame(game)) + + mod := domain.InstalledMod{Mod: domain.Mod{ID: "some-mod", SourceID: "nexusmods", Version: "1.0"}} + stale, err := svc.CheckBaseStaleness(game, []domain.InstalledMod{mod}) + require.NoError(t, err) + require.Empty(t, stale) +} + +// TestCheckGameUpdates_MergesStalenessWithoutDuplicating proves the +// combined seam CLI/TUI both use: a mod with a REAL update available is not +// separately duplicated as a staleness row even when it's also stale, and a +// mod with ONLY staleness (no real update) is still surfaced. +func TestCheckGameUpdates_MergesStalenessWithoutDuplicating(t *testing.T) { + svc, game, _ := newStalenessTestService(t) + + src := &updateMockSource{id: "fake-compiler", currentMod: &domain.Mod{ID: "has-real-update", Version: "2.0"}} + svc.RegisterSource(src) + + realUpdateMod := domain.InstalledMod{Mod: domain.Mod{ID: "has-real-update", SourceID: "fake-compiler", Version: "1.0"}} + staleOnlyMod := domain.InstalledMod{Mod: domain.Mod{ID: "stale-only", SourceID: "fake-compiler", Version: "1.0"}} + seedCompiledMod(t, svc, game, realUpdateMod, "0000000000000000000000000000000000dead") + seedCompiledMod(t, svc, game, staleOnlyMod, "0000000000000000000000000000000000dead") + + updates, err := svc.CheckGameUpdates(context.Background(), game, []domain.InstalledMod{realUpdateMod, staleOnlyMod}) + require.NoError(t, err) + require.Len(t, updates, 2, "one real-update row + one staleness-only row, no duplicate for the mod with both") + + byID := map[string]domain.Update{} + for _, u := range updates { + byID[u.InstalledMod.ID] = u + } + + real, ok := byID["has-real-update"] + require.True(t, ok) + require.Equal(t, "2.0", real.NewVersion) + require.False(t, real.RecompileNeeded, "a real version update supersedes the staleness row - recompiling happens as part of applying it") + + stale, ok := byID["stale-only"] + require.True(t, ok) + require.True(t, stale.RecompileNeeded) + require.Equal(t, "1.0", stale.NewVersion) +} diff --git a/internal/core/service_compile_fingerprint_test.go b/internal/core/service_compile_fingerprint_test.go new file mode 100644 index 0000000..c5778b4 --- /dev/null +++ b/internal/core/service_compile_fingerprint_test.go @@ -0,0 +1,129 @@ +package core_test + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/core" + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" + "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" + "github.com/stretchr/testify/require" +) + +// basePakIndexHash opens path and returns its footer IndexHash - the same +// value Service's compile branches record, computed independently here so +// tests can assert against it without depending on internal/core internals. +func basePakIndexHash(t *testing.T, path string) string { + t.Helper() + r, err := unrealpak.Open(path) + require.NoError(t, err) + defer r.Close() //nolint:errcheck + return r.IndexHash() +} + +// TestDownloadMod_DeployCompile_RecordsBaseIndexHashAndRetainedSource pins +// #196 design points 1-2 for the DOWNLOAD compile path: compiling an +// .exmodz must record the base pak's IndexHash under the file's real +// DownloadableFile.ID, retain the original .exmodz bytes beside the +// compiled pak, and keep both out of ListFiles/deploy. +func TestDownloadMod_DeployCompile_RecordsBaseIndexHashAndRetainedSource(t *testing.T) { + dlSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("original-exmodz-bytes")) + })) + defer dlSrv.Close() + + installDir := t.TempDir() + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + writeFakeBasePak(t, basePak) + wantHash := basePakIndexHash(t, basePak) + + cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} + svc, err := core.NewService(cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + src := &fakeCompilerSource{downloadURL: dlSrv.URL} + svc.RegisterSource(src) + + game := &domain.Game{ID: "icarus", InstallPath: installDir, ModPath: t.TempDir(), DeployMode: domain.DeployCompile} + require.NoError(t, svc.AddGame(game)) + + mod := &domain.Mod{ID: "bear-mount", SourceID: "fake-compiler", GameID: "icarus", Version: "3.3"} + file := &domain.DownloadableFile{ID: "exmodz-file-id", FileName: "Bear_Mount.exmodz"} + + _, err = svc.DownloadMod(context.Background(), "fake-compiler", game, mod, file, nil) + require.NoError(t, err) + + gameCache := svc.GetGameCache(game) + + hashes, err := gameCache.BaseIndexHashes(game.ID, mod.SourceID, mod.ID, mod.Version) + require.NoError(t, err) + require.Equal(t, map[string]string{"exmodz-file-id": wantHash}, hashes) + + retainedPath := gameCache.GetFilePath(game.ID, mod.SourceID, mod.ID, mod.Version, cache.RetainedSourceName("exmodz-file-id")) + retainedData, err := os.ReadFile(retainedPath) + require.NoError(t, err) + require.Equal(t, "original-exmodz-bytes", string(retainedData)) + + files, err := gameCache.ListFiles(game.ID, mod.SourceID, mod.ID, mod.Version) + require.NoError(t, err) + require.Equal(t, []string{"Bear_Mount_P.pak"}, files, "retained source and base-index marker must never be deployable content") +} + +// TestImportMod_DeployCompile_RecordsBaseIndexHashAndRetainedSource mirrors +// the above for the IMPORT compile path (keyed by the compiled output's own +// filename, since Import has no real DownloadableFile.ID available at +// compile time - see stageCompileFingerprint's doc comment). +func TestImportMod_DeployCompile_RecordsBaseIndexHashAndRetainedSource(t *testing.T) { + installDir := t.TempDir() + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + writeFakeBasePak(t, basePak) + wantHash := basePakIndexHash(t, basePak) + + cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} + svc, err := core.NewService(cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + src := &fakeCompilerSource{} + svc.RegisterSource(src) + + game := &domain.Game{ + ID: "icarus", + InstallPath: installDir, + ModPath: t.TempDir(), + DeployMode: domain.DeployCompile, + SourceIDs: map[string]string{"fake-compiler": "external-icarus-id"}, + } + require.NoError(t, svc.AddGame(game)) + + tempDir := t.TempDir() + archivePath := filepath.Join(tempDir, "Bear_Mount.exmodz") + require.NoError(t, os.WriteFile(archivePath, []byte("original-exmodz-bytes"), 0o644)) + + importer := svc.NewImporter(game) + result, err := importer.Import(context.Background(), archivePath, game, core.ImportOptions{}) + require.NoError(t, err) + + gameCache := svc.GetGameCache(game) + + hashes, err := gameCache.BaseIndexHashes(game.ID, result.Mod.SourceID, result.Mod.ID, result.Mod.Version) + require.NoError(t, err) + require.Equal(t, map[string]string{"Bear_Mount_P.pak": wantHash}, hashes) + + retainedPath := gameCache.GetFilePath(game.ID, result.Mod.SourceID, result.Mod.ID, result.Mod.Version, cache.RetainedSourceName("Bear_Mount_P.pak")) + retainedData, err := os.ReadFile(retainedPath) + require.NoError(t, err) + require.Equal(t, "original-exmodz-bytes", string(retainedData)) + + files, err := gameCache.ListFiles(game.ID, result.Mod.SourceID, result.Mod.ID, result.Mod.Version) + require.NoError(t, err) + require.Equal(t, []string{"Bear_Mount_P.pak"}, files, "retained source and base-index marker must never be deployable content") +} diff --git a/internal/core/service_icarus_compile_test.go b/internal/core/service_icarus_compile_test.go index 9467438..5dd29aa 100644 --- a/internal/core/service_icarus_compile_test.go +++ b/internal/core/service_icarus_compile_test.go @@ -11,9 +11,23 @@ import ( "github.com/DonovanMods/linux-mod-manager/internal/core" "github.com/DonovanMods/linux-mod-manager/internal/domain" "github.com/DonovanMods/linux-mod-manager/internal/source" + "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" "github.com/stretchr/testify/require" ) +// writeFakeBasePak writes a real, minimal but VALID pak at path (#196: the +// compile branch now opens the base pak itself to read its footer IndexHash +// as a compile fingerprint, so a bare byte-stub file - fine when only the +// fake compiler ever touched this path - no longer parses). One tiny table +// entry is enough; its content is never asserted on by these tests. +func writeFakeBasePak(t *testing.T, path string) { + t.Helper() + w, err := unrealpak.Create(path) + require.NoError(t, err) + require.NoError(t, w.AddFile("Data/D_Fixture.json", []byte(`{"fixture":true}`))) + require.NoError(t, w.Close()) +} + // fakeCompilerSource is a minimal ModSource that also implements // source.Compiler, standing in for internal/source/icarus.Icarus (Tasks // 8/13) without pulling that package into internal/core's tests — this test @@ -76,7 +90,7 @@ func TestDownloadMod_DeployCompile_InvokesCompiler(t *testing.T) { installDir := t.TempDir() basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) - require.NoError(t, os.WriteFile(basePak, []byte("fake-base-pak"), 0o644)) + writeFakeBasePak(t, basePak) cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} svc, err := core.NewService(cfg) @@ -123,7 +137,7 @@ func newCompileTestGame(t *testing.T, dlBody string) (*core.Service, *fakeCompil installDir := t.TempDir() basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) - require.NoError(t, os.WriteFile(basePak, []byte("fake-base-pak"), 0o644)) + writeFakeBasePak(t, basePak) cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} svc, err := core.NewService(cfg) diff --git a/internal/core/service_import_compile_test.go b/internal/core/service_import_compile_test.go index 8eb0512..6f32564 100644 --- a/internal/core/service_import_compile_test.go +++ b/internal/core/service_import_compile_test.go @@ -61,7 +61,7 @@ func newImportCompileTestGame(t *testing.T) (*core.Service, *fakeCompilerSource, installDir := t.TempDir() basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) - require.NoError(t, os.WriteFile(basePak, []byte("fake-base-pak"), 0o644)) + writeFakeBasePak(t, basePak) cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} svc, err := core.NewService(cfg) @@ -199,7 +199,7 @@ func TestImportMod_DeployCompile_NoCompilerSourceFailsLoud(t *testing.T) { installDir := t.TempDir() basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) - require.NoError(t, os.WriteFile(basePak, []byte("fake-base-pak"), 0o644)) + writeFakeBasePak(t, basePak) cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} svc, err := core.NewService(cfg) @@ -268,7 +268,7 @@ func TestImportMod_DeployCompile_CompileFailureLeavesNoPartialArtifact(t *testin installDir := t.TempDir() basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) - require.NoError(t, os.WriteFile(basePak, []byte("fake-base-pak"), 0o644)) + writeFakeBasePak(t, basePak) cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} svc, err := core.NewService(cfg) @@ -337,7 +337,7 @@ func TestImportMod_DeployCompile_ReimportSurvivesStagingFailure(t *testing.T) { installDir := t.TempDir() basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) - require.NoError(t, os.WriteFile(basePak, []byte("fake-base-pak"), 0o644)) + writeFakeBasePak(t, basePak) cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} svc, err := core.NewService(cfg) diff --git a/internal/core/updater.go b/internal/core/updater.go index 4922b2e..06691ed 100644 --- a/internal/core/updater.go +++ b/internal/core/updater.go @@ -147,3 +147,97 @@ func CompareVersions(v1, v2 string) int { func IsNewerVersion(currentVersion, newVersion string) bool { return domain.IsNewerVersion(currentVersion, newVersion) } + +// CheckBaseStaleness scans installed for DeployCompile mods whose compiled +// artifact no longer matches game's live base data.pak IndexHash (#196, +// "the Friday problem": a weekly base-pak refresh silently reverts the +// tables a compiled mod patches, and nothing before #196 ever noticed). +// This is entirely local/offline - unlike Updater.CheckUpdates it never +// contacts a source, so it runs for EVERY installed mod regardless of +// UpdatePolicy or SourceID, including pinned and domain.SourceLocal mods +// (pinning fixes the mod version, not the base pak; a pure local import has +// no remote to check version-wise but can still go stale against the base +// pak - see #196 design point 3). +// +// Only entries carrying the #196 fingerprint (cache.BaseIndexHashes) +// participate: a missing fingerprint is treated as "not a compiled entry we +// can reason about", not "stale" - a game whose DeployMode is DeployCompile +// can still legitimately serve prebuilt, never-compiled .pak files (see +// isExmodzFile's doc comment), and there is no local signal to tell those +// apart from a compiled entry that merely predates #196. (Design point 4's +// literal "missing fingerprint = always stale" was deliberately narrowed +// during #196 review to avoid exactly that false positive - see the PR +// description.) Any mod actually compiled under #196 always carries the +// fingerprint by construction (stageCompileFingerprint writes it in the +// SAME atomic commit as the compiled pak), so this narrowing costs nothing +// for anything compiled going forward. +func (s *Service) CheckBaseStaleness(game *domain.Game, installed []domain.InstalledMod) ([]domain.Update, error) { + if game.DeployMode != domain.DeployCompile { + return nil, nil + } + basePakPath, err := resolveBasePak(game) + if err != nil { + // No installed base pak to compare against: nothing to report this + // pass, not an error - matches CheckUpdates' own per-source + // tolerance for a signal that simply isn't available right now. + return nil, nil //nolint:nilerr + } + liveHash, err := basePakIndexHash(basePakPath) + if err != nil { + return nil, fmt.Errorf("reading base pak for staleness check: %w", err) + } + + gameCache := s.GetGameCache(game) + var stale []domain.Update + for _, mod := range installed { + hashes, err := gameCache.BaseIndexHashes(game.ID, mod.SourceID, mod.ID, mod.Version) + if err != nil { + continue // unreadable bookkeeping: silently skip, matching FileManifests' own tolerance + } + for _, recorded := range hashes { + if recorded != liveHash { + stale = append(stale, domain.Update{InstalledMod: mod, NewVersion: mod.Version, RecompileNeeded: true}) + break + } + } + } + return stale, nil +} + +// CheckGameUpdates is the single seam CLI and TUI both check updates +// through (#196): it combines Updater.CheckUpdates' remote version checks +// with CheckBaseStaleness' local base-pak staleness scan, so "does this mod +// need attention" means the same thing in both interfaces. A mod that +// already has a real version update available is not separately reported +// as stale even if it is: applying that update recompiles it fresh against +// the CURRENT base pak as a normal side effect of the compile step, so +// there is nothing left to flag once the real update lands. +// +// Errors from either half are tolerated the same way CheckUpdates already +// tolerates a single source failing: whatever updates were found are still +// returned, with the first non-nil error surfaced (checkErr takes priority +// as the richer, multi-source diagnostic when both fail). +func (s *Service) CheckGameUpdates(ctx context.Context, game *domain.Game, installed []domain.InstalledMod) ([]domain.Update, error) { + updates, checkErr := s.NewUpdater().CheckUpdates(ctx, game, installed) + + stale, staleErr := s.CheckBaseStaleness(game, installed) + if staleErr != nil && checkErr == nil { + checkErr = staleErr + } + + if len(stale) > 0 { + reported := make(map[string]bool, len(updates)) + for _, u := range updates { + reported[domain.ModKey(u.InstalledMod.SourceID, u.InstalledMod.ID)] = true + } + for _, u := range stale { + key := domain.ModKey(u.InstalledMod.SourceID, u.InstalledMod.ID) + if !reported[key] { + updates = append(updates, u) + reported[key] = true + } + } + } + + return updates, checkErr +} diff --git a/internal/domain/mod.go b/internal/domain/mod.go index 9552518..430a230 100644 --- a/internal/domain/mod.go +++ b/internal/domain/mod.go @@ -119,6 +119,14 @@ type Update struct { NewVersion string Changelog string FileIDReplacements map[string]string // Old file ID -> new file ID when a file was superseded (e.g. NexusMods FileUpdates) + // RecompileNeeded marks a DeployCompile mod whose deployed compile no + // longer matches the game's live base data.pak (#196, "the Friday + // problem": a weekly base-pak refresh silently reverts the tables a + // compiled mod patches, with nothing before #196 to notice). NewVersion + // equals InstalledMod.Version in this case - the mod itself hasn't + // changed, only the base pak has - so callers must not treat NewVersion + // as a real version bump when this is set. + RecompileNeeded bool } // ModKey returns a unique lookup key for a mod: "sourceID:modID". diff --git a/internal/storage/cache/cache.go b/internal/storage/cache/cache.go index b6d3b18..0307d9a 100644 --- a/internal/storage/cache/cache.go +++ b/internal/storage/cache/cache.go @@ -250,6 +250,84 @@ func (c *Cache) HasFileIDs(gameID, sourceID, modID, version string, fileIDs []st return true } +// baseIndexHashPrefix names a compiled file's base-pak fingerprint marker +// (#196, "the Friday problem"): the game's base data.pak footer IndexHash at +// the moment the file was compiled, so a later staleness check can detect +// the base pak changing underneath a compiled mod without re-hashing +// anything. Reserved (ReservedPrefix) so ListFiles/Size/deploy skip it like +// every other lmm bookkeeping entry. +const baseIndexHashPrefix = ReservedPrefix + "basehash-" + +// MarkBaseIndexHash records fileID's compiled-against base pak IndexHash +// (hex-encoded) into versionDir - written into the STAGING directory +// alongside the compile's own completion marker and retained source, just +// before the atomic commit that publishes all three together (mirrors +// MarkFileCompleteWithMembers; see internal/core's stageCompileFingerprint). +// An unverifiable fileID is skipped, matching writeFileMarker's contract. +func MarkBaseIndexHash(versionDir, fileID, indexHash string) error { + if !VerifiableFileID(fileID) { + return nil + } + if err := os.MkdirAll(versionDir, 0755); err != nil { + return fmt.Errorf("creating cache dir for base index marker: %w", err) + } + path := filepath.Join(versionDir, baseIndexHashPrefix+fileID) + if err := os.WriteFile(path, []byte(indexHash), 0644); err != nil { + return fmt.Errorf("writing base index marker: %w", err) + } + return nil +} + +// BaseIndexHashes reads every recorded base-pak fingerprint marker in the +// (gameID, sourceID, modID, version) cache entry, keyed by the compiled +// file's own fileID - mirroring FileManifests' directory-walk style so +// staleness detection (#196) works uniformly regardless of how a fileID was +// assigned (a download's real DownloadableFile.ID, or an import's synthetic +// one - see internal/core's stageCompileFingerprint). A directory with no +// compiled entries - including one with no markers at all - returns an +// empty map, never an error. +func (c *Cache) BaseIndexHashes(gameID, sourceID, modID, version string) (map[string]string, error) { + versionDir := c.ModPath(gameID, sourceID, modID, version) + entries, err := os.ReadDir(versionDir) + if err != nil { + if os.IsNotExist(err) { + return map[string]string{}, nil + } + return nil, fmt.Errorf("reading base index markers: %w", err) + } + + hashes := make(map[string]string) + for _, entry := range entries { + fileID, ok := strings.CutPrefix(entry.Name(), baseIndexHashPrefix) + if !ok || entry.IsDir() || !VerifiableFileID(fileID) { + continue + } + body, err := os.ReadFile(filepath.Join(versionDir, entry.Name())) + if err != nil { + return nil, fmt.Errorf("reading base index marker %s: %w", entry.Name(), err) + } + hashes[fileID] = string(body) + } + return hashes, nil +} + +// retainedSourcePrefix names a compiled file's retained source archive +// (#196): the original .exmodz kept beside the compiled pak so a later +// recompile - the base pak changed, not the mod - can run offline instead +// of re-downloading. Reserved (ReservedPrefix) so ListFiles/Size/deploy skip +// it like every other lmm bookkeeping entry: it is cache-internal +// provenance, never a deployment member. +const retainedSourcePrefix = ReservedPrefix + "source-" + +// RetainedSourceName returns the reserved on-disk filename for fileID's +// retained compile source. It is a pure naming function - like +// GetFilePath, callers join it against a staging or cache directory +// themselves and read/write/copy the actual bytes with ordinary file I/O +// (see internal/core's stageCompileFingerprint and recompile-apply path). +func RetainedSourceName(fileID string) string { + return retainedSourcePrefix + fileID +} + // Store saves a file to the cache func (c *Cache) Store(gameID, sourceID, modID, version, relativePath string, content []byte) error { modPath := c.ModPath(gameID, sourceID, modID, version) diff --git a/internal/storage/cache/cache_test.go b/internal/storage/cache/cache_test.go index e680be5..222fa13 100644 --- a/internal/storage/cache/cache_test.go +++ b/internal/storage/cache/cache_test.go @@ -3,6 +3,7 @@ package cache_test import ( "os" "path/filepath" + "strings" "testing" "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" @@ -474,3 +475,86 @@ func TestCache_MarkFileCompleteWithMembers_UnverifiableIDs(t *testing.T) { require.NoError(t, err) assert.Empty(t, entries, "an unverifiable file ID must not produce a marker") } + +// TestCache_BaseIndexHashes_RoundTrip mirrors TestCache_FileManifests_RoundTrip +// for the #196 base-pak fingerprint marker: write two, read them back keyed +// by fileID, and confirm a mod with none reports an empty map. +func TestCache_BaseIndexHashes_RoundTrip(t *testing.T) { + c := cache.New(t.TempDir()) + versionDir := c.ModPath("g", "src", "mod", "1.0") + + require.NoError(t, cache.MarkBaseIndexHash(versionDir, "file-a", "aaaa1111")) + require.NoError(t, cache.MarkBaseIndexHash(versionDir, "file-b", "bbbb2222")) + + hashes, err := c.BaseIndexHashes("g", "src", "mod", "1.0") + require.NoError(t, err) + assert.Equal(t, map[string]string{"file-a": "aaaa1111", "file-b": "bbbb2222"}, hashes) + + none, err := c.BaseIndexHashes("g", "src", "other-mod", "1.0") + require.NoError(t, err) + assert.Empty(t, none, "a version dir with no base-index markers reports an empty map, not an error") +} + +// TestCache_BaseIndexHashes_ExcludedFromContentEnumerators pins that base +// index markers are reserved bookkeeping (ReservedPrefix), never mod +// content: they must never be deployed, sized, or counted, exactly like +// completion markers (TestCache_ManifestMarkersStayReservedAndComplete). +func TestCache_BaseIndexHashes_ExcludedFromContentEnumerators(t *testing.T) { + c := cache.New(t.TempDir()) + + require.NoError(t, c.Store("g", "src", "mod", "1.0", "Bear_Mount_P.pak", []byte("12345"))) + require.NoError(t, cache.MarkBaseIndexHash(c.ModPath("g", "src", "mod", "1.0"), "file-a", "aaaa1111")) + + files, err := c.ListFiles("g", "src", "mod", "1.0") + require.NoError(t, err) + assert.Equal(t, []string{"Bear_Mount_P.pak"}, files, "base index markers must never be listed as content") + + size, err := c.Size("g", "src", "mod", "1.0") + require.NoError(t, err) + assert.Equal(t, int64(5), size, "base index marker bytes must not count toward cache size") +} + +// TestCache_MarkBaseIndexHash_UnverifiableIDs mirrors +// TestCache_MarkFileCompleteWithMembers_UnverifiableIDs: an unverifiable +// file ID is skipped rather than producing a marker. +func TestCache_MarkBaseIndexHash_UnverifiableIDs(t *testing.T) { + c := cache.New(t.TempDir()) + modPath := c.ModPath("g", "src", "mod", "1.0") + require.NoError(t, os.MkdirAll(modPath, 0755)) + + require.NoError(t, cache.MarkBaseIndexHash(modPath, "", "aaaa1111")) + require.NoError(t, cache.MarkBaseIndexHash(modPath, "../escape", "aaaa1111")) + + entries, err := os.ReadDir(modPath) + require.NoError(t, err) + assert.Empty(t, entries, "an unverifiable file ID must not produce a base index marker") +} + +// TestCache_RetainedSourceName_IsReservedAndExcludedFromContent pins that a +// retained compile source (#196) written under RetainedSourceName is +// reserved bookkeeping, not a deployment member - it must never be listed, +// sized, or deployed, even though its content is a real, non-empty file +// (unlike a marker, which is metadata). +func TestCache_RetainedSourceName_IsReservedAndExcludedFromContent(t *testing.T) { + name := cache.RetainedSourceName("file-a") + require.True(t, strings.HasPrefix(name, cache.ReservedPrefix), + "retained source name must live under the reserved namespace") + + c := cache.New(t.TempDir()) + require.NoError(t, c.Store("g", "src", "mod", "1.0", "Bear_Mount_P.pak", []byte("compiled"))) + require.NoError(t, c.Store("g", "src", "mod", "1.0", name, []byte("original exmodz bytes"))) + + files, err := c.ListFiles("g", "src", "mod", "1.0") + require.NoError(t, err) + assert.Equal(t, []string{"Bear_Mount_P.pak"}, files, "a retained source must never be listed as deployable content") + + size, err := c.Size("g", "src", "mod", "1.0") + require.NoError(t, err) + assert.Equal(t, int64(len("compiled")), size, "a retained source's bytes must not count toward cache size") +} + +// TestCache_RetainedSourceName_UniquePerFileID guards against two compiled +// files in the same mod entry colliding on their retained source's name. +func TestCache_RetainedSourceName_UniquePerFileID(t *testing.T) { + assert.NotEqual(t, cache.RetainedSourceName("file-a"), cache.RetainedSourceName("file-b")) +} diff --git a/internal/unrealpak/reader.go b/internal/unrealpak/reader.go index 818a04d..4960528 100644 --- a/internal/unrealpak/reader.go +++ b/internal/unrealpak/reader.go @@ -5,6 +5,7 @@ import ( "compress/zlib" "crypto/sha1" //nolint:gosec // pak format uses SHA1, not our choice "encoding/binary" + "encoding/hex" "fmt" "io" "math" @@ -22,6 +23,7 @@ type Reader struct { fileSize int64 // total size of the underlying file, for validateAllocSize methods [maxCompressionMethods]string // this pak's own CompressionMethods table mountPoint string // this pak's own primary-index MountPoint (see Writer's WithMountPoint) + indexHash [20]byte // this pak's own footer-recorded primary-index SHA1 (see IndexHash) } type readerEntry struct { @@ -71,7 +73,7 @@ func Open(path string) (*Reader, error) { return nil, fmt.Errorf("unrealpak: %s: parsing index: %w", path, err) } - return &Reader{f: f, entries: entries, fileSize: fileSize, methods: ft.methods, mountPoint: mountPoint}, nil + return &Reader{f: f, entries: entries, fileSize: fileSize, methods: ft.methods, mountPoint: mountPoint, indexHash: ft.indexHash}, nil } // methodName resolves a 1-based CompressionMethodIndex against this pak's own @@ -142,6 +144,15 @@ func (r *Reader) Close() error { return r.f.Close() } // form (see Writer's WithMountPoint, #178). func (r *Reader) MountPoint() string { return r.mountPoint } +// IndexHash returns the pak's footer-recorded primary-index SHA1 as a +// lowercase hex string — a cheap, stable fingerprint of the pak's content +// (Open already reads and verifies this region; IndexHash reads no further +// bytes and never hashes the pak's actual file payloads). Any content or +// layout change to the pak changes its primary index and therefore this +// hash, making it a reliable "has this pak changed" signal without the cost +// of hashing the whole (often multi-gigabyte) file. +func (r *Reader) IndexHash() string { return hex.EncodeToString(r.indexHash[:]) } + // Files returns every file this pak's index describes. func (r *Reader) Files() []FileEntry { out := make([]FileEntry, len(r.entries)) diff --git a/internal/unrealpak/reader_test.go b/internal/unrealpak/reader_test.go index 064fc92..9888fbd 100644 --- a/internal/unrealpak/reader_test.go +++ b/internal/unrealpak/reader_test.go @@ -124,6 +124,57 @@ func TestReader_Open_ListsFiles(t *testing.T) { } } +// IndexHash must be deterministic (same pak layout -> same hash) and must +// change when the pak's layout changes (#196: it's the staleness signal a +// base data.pak refresh is detected by). It must also never be all-zero - +// that would mean the footer's indexHash field was never actually captured. +// +// The primary index records each entry's path/offset/size, not its payload +// bytes - a same-length content edit that happens to leave every offset and +// size unchanged would NOT move IndexHash (this is the footer's own +// documented shape, not a gap in this method: see #196's design note that +// this is "cheap" specifically because it never reads payload data). B's +// content is a different LENGTH than A's so the size field actually differs, +// which is what a real pak rebuild's added/changed/removed rows always do. +func TestReader_IndexHash(t *testing.T) { + pathA := writeMinimalPak(t, "Icarus/Content/Data/Test.json", []byte(`{"a":1}`)) + pathA2 := writeMinimalPak(t, "Icarus/Content/Data/Test.json", []byte(`{"a":1}`)) + pathB := writeMinimalPak(t, "Icarus/Content/Data/Test.json", []byte(`{"a":22222}`)) + + rA, err := Open(pathA) + if err != nil { + t.Fatalf("Open A: %v", err) + } + defer rA.Close() //nolint:errcheck + rA2, err := Open(pathA2) + if err != nil { + t.Fatalf("Open A2: %v", err) + } + defer rA2.Close() //nolint:errcheck + rB, err := Open(pathB) + if err != nil { + t.Fatalf("Open B: %v", err) + } + defer rB.Close() //nolint:errcheck + + hashA := rA.IndexHash() + hashA2 := rA2.IndexHash() + hashB := rB.IndexHash() + + if len(hashA) != 40 { + t.Fatalf("IndexHash length = %d, want 40 (20-byte SHA1 hex)", len(hashA)) + } + if hashA == strings.Repeat("0", 40) { + t.Fatal("IndexHash is all-zero - footer indexHash was never captured") + } + if hashA != hashA2 { + t.Errorf("identical pak content produced different IndexHash: %q vs %q", hashA, hashA2) + } + if hashA == hashB { + t.Errorf("different pak content produced the same IndexHash: %q", hashA) + } +} + // A root-level file is keyed under the "/" directory in the directory index; // Files must report it without the leading slash, matching what hashPath uses. func TestReader_Open_RootLevelFile(t *testing.T) { From 50c4b86e49cccd4fdacbec0b5a3a9244b4bd48c2 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 18:44:48 -0400 Subject: [PATCH 58/96] feat: recompile a stale compile in place, lock/pin-aware (#196) Service.ApplyRecompile recompiles every compiled entry recorded for a mod against the game's CURRENT base pak, staged and committed atomically (this package's own staging pattern), then redeploys via Installer.ReplaceForUpdate with the mod's own file IDs on both sides of the transition (a no-op-shaped call that still refreshes a non-symlink deployment's stale on-disk bytes). - Recompiles offline from each file's retained .exmodz when present; falls back to re-downloading it when the retained copy is missing AND a real source/fileID connection exists (a download-compiled entry). An import-compiled entry or a domain.SourceLocal mod has no such connection and fails loud instead of guessing. - Locked mods are refused up front (ErrModLocked, same remedy wording as ApplyUpdate's own gate) - lock-wins, recompiling still rewrites the mod's deployed files even though its Version doesn't move. - Pinned mods recompile normally - pinning fixes the mod version, not the base pak. CLI/TUI wiring (update check/apply, verify warning) is next on this branch. --- internal/core/service_apply_recompile_test.go | 265 ++++++++++++++++++ internal/core/updater.go | 172 ++++++++++++ 2 files changed, 437 insertions(+) create mode 100644 internal/core/service_apply_recompile_test.go diff --git a/internal/core/service_apply_recompile_test.go b/internal/core/service_apply_recompile_test.go new file mode 100644 index 0000000..13bc790 --- /dev/null +++ b/internal/core/service_apply_recompile_test.go @@ -0,0 +1,265 @@ +package core_test + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/core" + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/source" + "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// redownloadCompilerSource wraps fakeCompilerSource with a real GetModFiles/ +// GetDownloadURL implementation backed by a local HTTP server, so +// ApplyRecompile's "retained source missing -> fall back to re-download" +// leg has something genuine to redownload from. +type redownloadCompilerSource struct { + *fakeCompilerSource + downloadBody string + files []domain.DownloadableFile + srv *httptest.Server +} + +func (s *redownloadCompilerSource) start(t *testing.T) { + t.Helper() + s.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(s.downloadBody)) + })) + t.Cleanup(s.srv.Close) +} + +func (s *redownloadCompilerSource) GetModFiles(ctx context.Context, mod *domain.Mod) ([]domain.DownloadableFile, error) { + return s.files, nil +} + +func (s *redownloadCompilerSource) GetDownloadURL(ctx context.Context, mod *domain.Mod, fileID string) (string, error) { + return s.srv.URL, nil +} + +var _ source.ModSource = (*redownloadCompilerSource)(nil) + +// recompileFixture bundles what ApplyRecompile's tests need to assert on: +// the service, game, deployed file's game-dir path, and the base pak path +// (so a test can rewrite it to simulate a base-pak refresh). +type recompileFixture struct { + svc *core.Service + game *domain.Game + deployedPath string // game.ModPath/Bear_Mount_P.pak + basePak string +} + +// seedCompiledInstalledMod builds a DeployCompile game with an installed, +// DEPLOYED compiled mod: cache holds the compiled pak, its retained source, +// and a (possibly stale) base-index marker; the mod is installed via the +// real Installer (so redeploy assertions exercise the real linker) and +// recorded in the DB/profile like any other install. linkMethod is caller- +// controlled because a symlink deployment would trivially reflect the +// atomic cache swap on its own - a copy/hardlink deployment is what proves +// ApplyRecompile's redeploy step actually ran. +func seedCompiledInstalledMod(t *testing.T, linkMethod domain.LinkMethod, sourceID string, recordedHash string) recompileFixture { + t.Helper() + + svc := newFlowsTestService(t) + installDir := t.TempDir() + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + writeFakeBasePak(t, basePak) + + game := &domain.Game{ + ID: "icarus", + InstallPath: installDir, + ModPath: t.TempDir(), + DeployMode: domain.DeployCompile, + LinkMethod: linkMethod, + SourceIDs: map[string]string{"fake-compiler": "external-icarus-id"}, + } + require.NoError(t, svc.AddGame(game)) + + const modID, version, fileID = "bear-mount", "3.3", "exmodz-file-id" + gameCache := svc.GetGameCache(game) + require.NoError(t, gameCache.Store(game.ID, sourceID, modID, version, "Bear_Mount_P.pak", []byte("stale-compiled-bytes"))) + require.NoError(t, gameCache.Store(game.ID, sourceID, modID, version, cache.RetainedSourceName(fileID), []byte("retained-exmodz-bytes"))) + versionDir := gameCache.ModPath(game.ID, sourceID, modID, version) + require.NoError(t, cache.MarkFileCompleteWithMembers(versionDir, fileID, []string{"Bear_Mount_P.pak"})) + if recordedHash != "" { + require.NoError(t, cache.MarkBaseIndexHash(versionDir, fileID, recordedHash)) + } + + im := &domain.InstalledMod{ + Mod: domain.Mod{ID: modID, SourceID: sourceID, Name: "Bear Mount", Version: version, GameID: game.ID}, + ProfileName: "default", + UpdatePolicy: domain.UpdateNotify, + Enabled: true, + Deployed: true, + LinkMethod: linkMethod, + FileIDs: []string{fileID}, + } + require.NoError(t, svc.SaveInstalledMod(im)) + + installer := svc.GetInstaller(game) + require.NoError(t, installer.Install(context.Background(), game, &im.Mod, "default")) + + pm := svc.NewProfileManager() + _, cerr := pm.Create(game.ID, "default") + require.NoError(t, cerr) + require.NoError(t, pm.UpsertMod(game.ID, "default", domain.ModReference{SourceID: sourceID, ModID: modID, Version: version, FileIDs: []string{fileID}})) + + return recompileFixture{svc: svc, game: game, deployedPath: filepath.Join(game.ModPath, "Bear_Mount_P.pak"), basePak: basePak} +} + +// TestApplyRecompile_OfflineFromRetainedSource_Redeploys is the happy path: +// a stale compile recompiles from its retained .exmodz with no network +// access at all (no source registered), lands the fresh bytes in the cache +// under the SAME name, records the live base pak's fingerprint, and +// redeploys - proven with LinkCopy so the on-disk deployed file can only +// carry the new content if ReplaceForUpdate actually ran. +func TestApplyRecompile_OfflineFromRetainedSource_Redeploys(t *testing.T) { + fx := seedCompiledInstalledMod(t, domain.LinkCopy, "fake-compiler", "0000000000000000000000000000000000dead") + liveHash := basePakIndexHash(t, fx.basePak) + + compiler := &fakeCompilerSource{} + fx.svc.RegisterSource(compiler) + + mod, err := fx.svc.GetInstalledMod("fake-compiler", "bear-mount", "icarus", "default") + require.NoError(t, err) + + result, err := fx.svc.ApplyRecompile(context.Background(), fx.game, "default", *mod, nil) + require.NoError(t, err) + require.Equal(t, []string{"Bear_Mount_P.pak"}, result.Applied) + require.Equal(t, 1, compiler.compileCalls) + + // fakeCompilerSource.Compile copies sourceFilePath's bytes through + // unchanged - the retained source's content, proving it (not a + // redownload) was used. + deployedData, err := os.ReadFile(fx.deployedPath) + require.NoError(t, err) + assert.Equal(t, "retained-exmodz-bytes", string(deployedData), "redeploy must reflect the freshly recompiled bytes") + + gameCache := fx.svc.GetGameCache(fx.game) + hashes, err := gameCache.BaseIndexHashes(fx.game.ID, "fake-compiler", "bear-mount", "3.3") + require.NoError(t, err) + assert.Equal(t, liveHash, hashes["exmodz-file-id"], "the recompile must record the CURRENT live base pak hash") +} + +// TestApplyRecompile_LockedRefRefuses mirrors +// TestApplyUpdate_LockedRefRefusesUpdate exactly (#196: lock-wins) - a +// locked mod's files must never be touched by a recompile. +func TestApplyRecompile_LockedRefRefuses(t *testing.T) { + fx := seedCompiledInstalledMod(t, domain.LinkCopy, "fake-compiler", "0000000000000000000000000000000000dead") + fx.svc.RegisterSource(&fakeCompilerSource{}) + + pm := fx.svc.NewProfileManager() + require.NoError(t, pm.SetModLock(fx.game.ID, "default", "fake-compiler", "bear-mount", "")) + + mod, err := fx.svc.GetInstalledMod("fake-compiler", "bear-mount", "icarus", "default") + require.NoError(t, err) + + before, err := os.ReadFile(fx.deployedPath) + require.NoError(t, err) + + _, err = fx.svc.ApplyRecompile(context.Background(), fx.game, "default", *mod, nil) + require.Error(t, err) + assert.ErrorIs(t, err, core.ErrModLocked) + assert.Contains(t, err.Error(), "locked at v") + + after, err := os.ReadFile(fx.deployedPath) + require.NoError(t, err) + assert.Equal(t, before, after, "a locked mod's deployed files must never be touched") +} + +// TestApplyRecompile_PinnedModRecompiles proves pinning does NOT block +// ApplyRecompile (#196 design point 3: pinning fixes the mod VERSION, not +// the base pak) - only ApplyUpdate/UpdateCheckable gate on UpdatePinned. +func TestApplyRecompile_PinnedModRecompiles(t *testing.T) { + fx := seedCompiledInstalledMod(t, domain.LinkCopy, "fake-compiler", "0000000000000000000000000000000000dead") + fx.svc.RegisterSource(&fakeCompilerSource{}) + + mod, err := fx.svc.GetInstalledMod("fake-compiler", "bear-mount", "icarus", "default") + require.NoError(t, err) + mod.UpdatePolicy = domain.UpdatePinned + + _, err = fx.svc.ApplyRecompile(context.Background(), fx.game, "default", *mod, nil) + require.NoError(t, err, "ApplyRecompile itself must not gate on UpdatePolicy") +} + +// TestApplyRecompile_LocalModMissingRetainedSource_FailsLoud: a pure local +// import has no remote to fall back to - a missing retained source must +// fail loud with an actionable remedy, never silently skip or fabricate +// content. +func TestApplyRecompile_LocalModMissingRetainedSource_FailsLoud(t *testing.T) { + fx := seedCompiledInstalledMod(t, domain.LinkCopy, domain.SourceLocal, "0000000000000000000000000000000000dead") + + gameCache := fx.svc.GetGameCache(fx.game) + retainedPath := gameCache.GetFilePath(fx.game.ID, domain.SourceLocal, "bear-mount", "3.3", cache.RetainedSourceName("exmodz-file-id")) + require.NoError(t, os.Remove(retainedPath)) + + fx.svc.RegisterSource(&fakeCompilerSource{}) + + mod, err := fx.svc.GetInstalledMod(domain.SourceLocal, "bear-mount", "icarus", "default") + require.NoError(t, err) + + _, err = fx.svc.ApplyRecompile(context.Background(), fx.game, "default", *mod, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "retained compile source") + assert.Contains(t, err.Error(), "no remote source") +} + +// TestApplyRecompile_MissingRetainedSource_FallsBackToRedownload proves the +// #196 design's "fallback: re-download" leg for a mod with a REAL source +// connection (a download-compiled entry: fileID is that source's actual +// DownloadableFile.ID, so GetModFiles/GetDownloadURL can resolve it). +func TestApplyRecompile_MissingRetainedSource_FallsBackToRedownload(t *testing.T) { + fx := seedCompiledInstalledMod(t, domain.LinkCopy, "fake-compiler", "0000000000000000000000000000000000dead") + + gameCache := fx.svc.GetGameCache(fx.game) + retainedPath := gameCache.GetFilePath(fx.game.ID, "fake-compiler", "bear-mount", "3.3", cache.RetainedSourceName("exmodz-file-id")) + require.NoError(t, os.Remove(retainedPath)) + + compiler := &redownloadCompilerSource{ + fakeCompilerSource: &fakeCompilerSource{}, + downloadBody: "redownloaded-exmodz-bytes", + files: []domain.DownloadableFile{{ID: "exmodz-file-id", FileName: "Bear_Mount.exmodz"}}, + } + compiler.start(t) + fx.svc.RegisterSource(compiler) + + mod, err := fx.svc.GetInstalledMod("fake-compiler", "bear-mount", "icarus", "default") + require.NoError(t, err) + + result, err := fx.svc.ApplyRecompile(context.Background(), fx.game, "default", *mod, nil) + require.NoError(t, err) + require.Equal(t, []string{"Bear_Mount_P.pak"}, result.Applied) + + deployedData, err := os.ReadFile(fx.deployedPath) + require.NoError(t, err) + assert.Equal(t, "redownloaded-exmodz-bytes", string(deployedData)) +} + +// TestApplyRecompile_NoCompiledEntries_FailsLoud: a mod with no base-index +// markers at all (never compiled) has nothing for ApplyRecompile to do - +// callers should never route such a mod here, but the gate must still fail +// loud rather than silently no-op if one slips through. +func TestApplyRecompile_NoCompiledEntries_FailsLoud(t *testing.T) { + svc := newFlowsTestService(t) + installDir := t.TempDir() + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + writeFakeBasePak(t, basePak) + + game := &domain.Game{ID: "icarus", InstallPath: installDir, ModPath: t.TempDir(), DeployMode: domain.DeployCompile} + require.NoError(t, svc.AddGame(game)) + svc.RegisterSource(&fakeCompilerSource{}) + + mod := domain.InstalledMod{Mod: domain.Mod{ID: "plain-pak-mod", SourceID: "fake-compiler", Name: "Plain Pak", Version: "1.0", GameID: "icarus"}} + + _, err := svc.ApplyRecompile(context.Background(), game, "default", mod, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "no compiled entries") +} diff --git a/internal/core/updater.go b/internal/core/updater.go index 06691ed..6e1af61 100644 --- a/internal/core/updater.go +++ b/internal/core/updater.go @@ -4,9 +4,12 @@ import ( "context" "errors" "fmt" + "os" + "path/filepath" "github.com/DonovanMods/linux-mod-manager/internal/domain" "github.com/DonovanMods/linux-mod-manager/internal/source" + "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" ) // Updater checks for and applies mod updates @@ -241,3 +244,172 @@ func (s *Service) CheckGameUpdates(ctx context.Context, game *domain.Game, insta return updates, checkErr } + +// ApplyRecompile recompiles mod IN PLACE at its CURRENT version against +// game's live base pak (#196: a base data.pak refresh left its already- +// deployed compile(s) patching stale tables - "the Friday problem"). +// Every compiled entry recorded for mod (cache.BaseIndexHashes) is +// recompiled, not just the ones CheckBaseStaleness found mismatched - a +// mod's compiled files always share one base pak, so a partial refresh +// would leave the entry internally inconsistent for no benefit. +// +// Recompiles from each file's retained .exmodz (offline) when present; +// falls back to re-downloading it from mod's source when the retained copy +// is missing AND a real fileID/source connection exists (a download- +// compiled entry). An import-compiled entry has no such connection (Import +// resolves no real source file ID - see stageCompileFingerprint) and a +// domain.SourceLocal mod has no source at all either way: for both, a +// missing retained source fails loud naming the fix (re-import/re-add the +// mod) rather than guessing. +// +// Locked mods are refused outright before any work happens - mirrors +// ApplyUpdate's own lock gate exactly (lock-wins: recompiling still +// rewrites the mod's deployed FILES even though its Version doesn't move, +// which is exactly what a lock forbids). Pinned mods ARE recompiled - +// pinning fixes the mod's VERSION, not the base pak (#196 design point 3); +// callers must not route a pinned mod through this gate at all. +// +// The cache update is staged and committed atomically (this package's own +// staging/commit pattern - prepareStaging + commitStagedCache), so a +// mid-recompile failure never touches the existing good entry. Redeploy +// reuses Installer.ReplaceForUpdate with the mod's OWN file IDs on both +// sides of the transition: with nothing IDs-wise changing, it degrades to +// its historical union-replace behavior, which simply re-links/re-copies +// every current member - exactly what refreshes a non-symlink deployment's +// stale on-disk bytes (a symlink deployment already reflects the new cache +// content once the atomic swap above lands, so this step is a correctness +// no-op for it, not a wasted one). +func (s *Service) ApplyRecompile(ctx context.Context, game *domain.Game, profileName string, mod domain.InstalledMod, progress func(DeployProgress)) (result *UpdateApplyResult, err error) { + result = &UpdateApplyResult{} + emit := func(p DeployProgress) { + if progress != nil { + progress(p) + } + } + base := DeployProgress{ModName: mod.Name, ModID: mod.ID, SourceID: mod.SourceID} + + if prof, perr := s.NewProfileManager().Get(game.ID, profileName); perr == nil { + if ref := prof.FindRef(mod.SourceID, mod.ID); ref != nil && ref.Locked { + return result, LockedRefRefusalError(mod.Mod, profileName, ref) + } + } + + basePakPath, err := resolveBasePak(game) + if err != nil { + return result, err + } + + gameCache := s.GetGameCache(game) + hashes, err := gameCache.BaseIndexHashes(game.ID, mod.SourceID, mod.ID, mod.Version) + if err != nil { + return result, fmt.Errorf("reading compile fingerprints: %w", err) + } + if len(hashes) == 0 { + return result, fmt.Errorf("%s has no compiled entries to recompile", mod.Name) + } + + compiler, err := s.compilerSourceForGame(game.ID) + if err != nil { + return result, err + } + + manifests, err := gameCache.FileManifests(game.ID, mod.SourceID, mod.ID, mod.Version) + if err != nil { + return result, fmt.Errorf("reading cache manifests: %w", err) + } + + cacheModRef := &domain.Mod{ID: mod.ID, SourceID: mod.SourceID, Version: mod.Version, GameID: game.ID} + cachePath, stagePath, err := prepareStaging(gameCache, game, cacheModRef) + if err != nil { + return result, err + } + defer os.RemoveAll(stagePath) //nolint:errcheck + if err := os.MkdirAll(stagePath, 0755); err != nil { + return result, fmt.Errorf("preparing recompile staging: %w", err) + } + + // Lazily fetched: the common path (retained source present for every + // compiled file) never needs the source's file listing at all. + var sourceFiles []domain.DownloadableFile + var sourceFilesErr error + getSourceFiles := func() ([]domain.DownloadableFile, error) { + if sourceFiles == nil && sourceFilesErr == nil { + sourceFiles, sourceFilesErr = s.GetModFiles(ctx, mod.SourceID, &mod.Mod) + } + return sourceFiles, sourceFilesErr + } + + for fileID := range hashes { + // destName is the compiled output's own filename: for a download- + // compiled entry it's the recorded manifest member; for an import- + // compiled entry (no manifest - see importer.go's compile branch) + // fileID already IS that name (stageCompileFingerprint's doc + // comment), so the fallback is exact, not a guess. + destName := fileID + if m, ok := manifests[fileID]; ok && m.Recorded && len(m.Members) == 1 { + destName = m.Members[0] + } + + retainedPath := gameCache.GetFilePath(game.ID, mod.SourceID, mod.ID, mod.Version, cache.RetainedSourceName(fileID)) + sourcePath := retainedPath + if _, statErr := os.Stat(retainedPath); statErr != nil { + if mod.SourceID == domain.SourceLocal { + return result, fmt.Errorf("%s: retained compile source for %q is missing and this mod has no remote source to re-download from - re-import the .exmodz to restore it", mod.Name, fileID) + } + files, ferr := getSourceFiles() + if ferr != nil { + return result, fmt.Errorf("%s: retained compile source for %q is missing; fetching source files: %w", mod.Name, fileID, ferr) + } + var match *domain.DownloadableFile + for i := range files { + if files[i].ID == fileID { + match = &files[i] + break + } + } + if match == nil { + return result, fmt.Errorf("%s: retained compile source for %q is missing and no matching source file was found to re-download", mod.Name, fileID) + } + url, uerr := s.GetDownloadURL(ctx, mod.SourceID, &mod.Mod, match.ID) + if uerr != nil { + return result, fmt.Errorf("%s: re-downloading compile source: %w", mod.Name, uerr) + } + tempDir, terr := newStagingDir(s.stagingRoot(), "lmm-recompile-*") + if terr != nil { + return result, terr + } + defer os.RemoveAll(tempDir) //nolint:errcheck + dlPath := filepath.Join(tempDir, match.FileName) + evt := base + evt.Phase, evt.Detail = UpdateNote, fmt.Sprintf("retained compile source missing for %s - re-downloading", destName) + emit(evt) + if _, derr := s.downloader.DownloadWithHeaders(ctx, url, dlPath, nil, nil); derr != nil { + return result, fmt.Errorf("%s: re-downloading compile source: %w", mod.Name, derr) + } + sourcePath = dlPath + } + + outPath := filepath.Join(stagePath, destName) + if cerr := compiler.Compile(ctx, basePakPath, sourcePath, outPath); cerr != nil { + return result, fmt.Errorf("recompiling %s: %w", destName, cerr) + } + if ferr := stageCompileFingerprint(stagePath, fileID, basePakPath, sourcePath); ferr != nil { + return result, ferr + } + result.Applied = append(result.Applied, destName) + } + + if err := commitStagedCache(cachePath, stagePath); err != nil { + return result, err + } + + installer, err := s.GetInstallerForProfile(game, profileName) + if err != nil { + return result, fmt.Errorf("recompiled %s but could not redeploy: %w", mod.Name, err) + } + if err := installer.ReplaceForUpdate(ctx, game, &mod.Mod, &mod.Mod, profileName, mod.FileIDs, mod.FileIDs); err != nil { + return result, fmt.Errorf("recompiled %s but redeploying failed: %w", mod.Name, err) + } + + return result, nil +} From b470a0b77e227520f3c5e367bc0dcb36cf691ed8 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 18:49:14 -0400 Subject: [PATCH 59/96] feat: wire base-pak recompile through lmm update (#196) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lmm update's check phase now goes through Service.CheckGameUpdates (remote version checks + local base-pak staleness), and apply dispatches a RecompileNeeded row to Service.ApplyRecompile instead of ApplyUpdate - both the bulk auto/--all loop and the single-mod `lmm update ` path. - Bulk table: a staleness row gets a "[recompile]" POLICY marker alongside the existing "[locked@version]" one. - Bulk --json: updateModJSON gains additive recompile_needed/reason fields (locked/available/etc. fields unchanged - JSON-contract- additions-are-MINOR precedent, #143/#155). - Single-mod: a staleness row prints "Recompiling %s (base pak updated)..." instead of a misleading same-version "Updating vX → vX", and --json reports a new "recompiled"/"recompile_available" status with ToVersion == FromVersion. - A locked staleness row is refused with the same lock-wins wording as a real update, before any work happens. make man regenerated (updateCmd.Long documents the new JSON shapes). verify's stale-compile warning and TUI parity are next on this branch. --- cmd/lmm/update.go | 128 +++++++++++++++++++-- cmd/lmm/update_recompile_test.go | 191 +++++++++++++++++++++++++++++++ docs/man/man1/lmm-update.1 | 17 ++- 3 files changed, 321 insertions(+), 15 deletions(-) create mode 100644 cmd/lmm/update_recompile_test.go diff --git a/cmd/lmm/update.go b/cmd/lmm/update.go index 756c1c4..0024932 100644 --- a/cmd/lmm/update.go +++ b/cmd/lmm/update.go @@ -64,6 +64,16 @@ type updateModJSON struct { // the lock moves or clears, even under auto policy or --all. Omitted // (not false) when unlocked, so pre-#143 documents are unchanged. Locked bool `json:"locked,omitempty"` + // RecompileNeeded is the --json sibling of the bulk table's "[recompile]" + // POLICY marker (#196): true means this row is a base-pak staleness + // signal, not a real mod version update - Available equals Current, and + // Reason explains why. Omitted (not false) when the row is a normal + // update, so pre-#196 documents are unchanged (additive field, per the + // JSON-contract-additions-are-MINOR precedent - see #143/#155). + RecompileNeeded bool `json:"recompile_needed,omitempty"` + // Reason qualifies RecompileNeeded: "stale_compile" today. Omitted + // otherwise. + Reason string `json:"reason,omitempty"` } // singleUpdateJSON is the one-document --json result of `lmm update ` @@ -81,6 +91,9 @@ type singleUpdateJSON struct { ToVersion string `json:"to_version,omitempty"` // omitted for up_to_date / skipped Changelog string `json:"changelog,omitempty"` // Status: "updated" | "up_to_date" | "skipped" | "available" | "rolled_back" + // | "recompiled" | "recompile_available" (#196: a base-pak staleness row, + // applied or --dry-run respectively - ToVersion equals FromVersion for + // both, since the mod itself hasn't changed). Status string `json:"status"` // Reason qualifies status=="skipped": "pinned" | "local" | "locked". // Omitted otherwise. @@ -116,18 +129,26 @@ If the update check itself fails partway through (e.g. a source outage), whatever was learned before the failure is still printed and the command exits non-zero rather than silently claiming success. +For a compile-deploy game (e.g. Icarus), a compiled mod whose game base pak +has changed since it was last compiled ("recompile needed") is checked and +applied through this same command: no new version, just a same-version +recompile against the current base pak. + --json prints exactly one JSON document to stdout, in one of two shapes: - Bulk check (no mod ID): {game_id, profile, updates: [...], skipped: {pinned, local}, error?}. error is present when the check itself failed partway through; updates/skipped still reflect whatever was learned first. A locked mod's updates[] entry carries "locked": true (omitted when unlocked): the update is reported but will not be - applied until the lock moves or clears. + applied until the lock moves or clears. A recompile-needed entry + carries "recompile_needed": true and "reason": "stale_compile" + (available_version equals current_version - the mod hasn't changed). - Single mod (a mod ID given) or 'update rollback': {mod_id, name, from_version, to_version, changelog, status, reason}. status is one - of "updated", "up_to_date", "skipped", "available" (--dry-run), or - "rolled_back"; reason is set only when status is "skipped" ("pinned", - "local", or "locked"). + of "updated", "up_to_date", "skipped", "available" (--dry-run), + "rolled_back", "recompiled", or "recompile_available" (--dry-run, + same-version base-pak recompile); reason is set only when status is + "skipped" ("pinned", "local", or "locked"). Examples: lmm update --game skyrim-se # Check all mods for updates @@ -292,9 +313,10 @@ func doUpdate(ctx context.Context, service *core.Service, game *domain.Game, arg })) } - // Check for updates (partial results returned even when some mods fail to fetch) - updater := service.NewUpdater() - updates, checkErr := updater.CheckUpdates(ctx, game, installed) + // Check for updates (partial results returned even when some mods fail to + // fetch) plus, for DeployCompile games, base-pak staleness (#196) - + // CheckGameUpdates is the single seam CLI and TUI both check through. + updates, checkErr := service.CheckGameUpdates(ctx, game, installed) if checkErr != nil { if errors.Is(checkErr, domain.ErrAuthRequired) { return authPromptError(updateSource) @@ -381,7 +403,7 @@ func doUpdate(ctx context.Context, service *core.Service, game *domain.Game, arg } for i, u := range updates { _, isLocked := lockedRefs[domain.ModKey(u.InstalledMod.SourceID, u.InstalledMod.ID)] - out.Updates[i] = updateModJSON{ + row := updateModJSON{ ModID: u.InstalledMod.ID, Name: u.InstalledMod.Name, Current: u.InstalledMod.Version, @@ -389,6 +411,11 @@ func doUpdate(ctx context.Context, service *core.Service, game *domain.Game, arg UpdatePolicy: policyToString(u.InstalledMod.UpdatePolicy), Locked: isLocked, } + if u.RecompileNeeded { + row.RecompileNeeded = true + row.Reason = "stale_compile" + } + out.Updates[i] = row } enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") @@ -421,6 +448,12 @@ func doUpdate(ctx context.Context, service *core.Service, game *domain.Game, arg if isLocked { policyStr += " [locked@" + lockedVersion + "]" } + if update.RecompileNeeded { + // #196: a base-pak staleness row - Available above equals + // Current, so this marker is the only thing telling it apart + // from a genuine no-op row in the table. + policyStr += " [recompile]" + } if update.InstalledMod.UpdatePolicy == domain.UpdateAuto { if isLocked { lockedAuto++ @@ -564,9 +597,8 @@ func applySingleUpdate(ctx context.Context, service *core.Service, game *domain. } } - // Check for update for this specific mod - updater := service.NewUpdater() - updates, err := updater.CheckUpdates(ctx, game, []domain.InstalledMod{*mod}) + // Check for update for this specific mod (plus base-pak staleness, #196) + updates, err := service.CheckGameUpdates(ctx, game, []domain.InstalledMod{*mod}) if err != nil { if errors.Is(err, domain.ErrAuthRequired) { return authPromptError(updateSource) @@ -606,6 +638,50 @@ func applySingleUpdate(ctx context.Context, service *core.Service, game *domain. } update := updates[0] + + // #196: a base-pak staleness row carries no real version change + // (NewVersion == mod.Version) - branch off before any of the + // version-bump wording/JSON below, which would otherwise print a + // misleading "Updating vX → vX...". + if update.RecompileNeeded { + if locked { + if jsonOutput { + return emitSingleUpdateJSON(singleUpdateJSON{ + ModID: mod.ID, Name: mod.Name, FromVersion: mod.Version, ToVersion: mod.Version, Status: "skipped", Reason: "locked", + }) + } + fmt.Printf("Recompile needed for %s (base pak updated) — but it is locked at v%s.\n", mod.Name, lockedVersion) + fmt.Printf("Move the lock: lmm mod lock -s %s -p %s %s %s | Unlock: lmm mod unlock -s %s -p %s %s\n", mod.SourceID, profileName, mod.ID, mod.Version, mod.SourceID, profileName, mod.ID) + return nil + } + + if !jsonOutput { + fmt.Printf("Recompiling %s (base pak updated)...\n", mod.Name) + } + + if updateDryRun { + if jsonOutput { + return emitSingleUpdateJSON(singleUpdateJSON{ + ModID: mod.ID, Name: mod.Name, FromVersion: mod.Version, ToVersion: mod.Version, Status: "recompile_available", + }) + } + fmt.Println("(dry-run: no changes applied)") + return nil + } + + if err := applyRecompile(ctx, service, game, *mod, profileName); err != nil { + return err + } + + if jsonOutput { + return emitSingleUpdateJSON(singleUpdateJSON{ + ModID: mod.ID, Name: mod.Name, FromVersion: mod.Version, ToVersion: mod.Version, Status: "recompiled", + }) + } + fmt.Printf("\n%s Recompiled: %s (base pak updated)\n", colorGreen("✓"), mod.Name) + return nil + } + oldVersion := mod.Version newVersion := update.NewVersion @@ -682,7 +758,16 @@ func applySingleUpdate(ctx context.Context, service *core.Service, game *domain. // core.ApplyUpdate's progress events - reproducing the pre-extraction CLI's // exact console positioning (download progress, forced-hook warnings, // after_each hook warnings, and the --verbose-gated link-method note). +// +// #196: a RecompileNeeded row carries no real version change (NewVersion == +// InstalledMod.Version) - it is routed to Service.ApplyRecompile instead, +// which has no hooks to run and no version/FileIDs to record, only the +// recompile-and-redeploy step itself. func applyUpdate(ctx context.Context, service *core.Service, game *domain.Game, upd domain.Update, profileName string) error { + if upd.RecompileNeeded { + return applyRecompile(ctx, service, game, upd.InstalledMod, profileName) + } + opts := core.UpdateOptions{ Hooks: getResolvedHooks(service, game, profileName), HookRunner: getHookRunner(service), @@ -713,6 +798,27 @@ func applyUpdate(ctx context.Context, service *core.Service, game *domain.Game, return err } +// applyRecompile applies a #196 base-pak staleness row via +// Service.ApplyRecompile, printing from its progress events the same way +// applyUpdate does for its own (UpdateWarning/UpdateNote are the only +// phases ApplyRecompile emits - it runs no hooks and downloads nothing +// worth a progress bar). +func applyRecompile(ctx context.Context, service *core.Service, game *domain.Game, mod domain.InstalledMod, profileName string) error { + progress := func(p core.DeployProgress) { + switch p.Phase { + case core.UpdateWarning: + fmt.Fprintf(os.Stderr, "Warning: %s\n", p.Detail) + case core.UpdateNote: + if verbose && !jsonOutput { + fmt.Printf(" %s\n", p.Detail) + } + } + } + + _, err := service.ApplyRecompile(ctx, game, profileName, mod, progress) + return err +} + func runUpdateRollback(cmd *cobra.Command, args []string) error { return withGameService(cmd, func(ctx context.Context, service *core.Service, game *domain.Game) error { return doUpdateRollback(ctx, service, game, args[0]) diff --git a/cmd/lmm/update_recompile_test.go b/cmd/lmm/update_recompile_test.go new file mode 100644 index 0000000..3ffc839 --- /dev/null +++ b/cmd/lmm/update_recompile_test.go @@ -0,0 +1,191 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/core" + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// setupDoUpdateRecompileTest builds a DeployCompile game with a registered +// compiler-capable source and an installed, deployed compiled mod whose +// recorded base-pak fingerprint is deliberately wrong, so `lmm update` +// reports/applies a #196 recompile row end to end through the CLI. +// linkMethod is LinkCopy so a successful recompile+redeploy is provable +// from the on-disk deployed bytes (a symlink would trivially reflect an +// in-place cache swap on its own). +func setupDoUpdateRecompileTest(t *testing.T) (*core.Service, *domain.Game, *compilerInstallSource, string) { + t.Helper() + + configDir = t.TempDir() + dataDir = t.TempDir() + + installDir := t.TempDir() + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + writeFakeBasePak(t, basePak) + + svc, err := core.NewService(core.ServiceConfig{ConfigDir: configDir, DataDir: dataDir, CacheDir: t.TempDir()}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + compiler := &compilerInstallSource{fakeInstallSource: newFakeInstallSource("fake-compiler")} + svc.RegisterSource(compiler) + + game := &domain.Game{ + ID: "icarus", Name: "Icarus", InstallPath: installDir, ModPath: t.TempDir(), + DeployMode: domain.DeployCompile, LinkMethod: domain.LinkCopy, + SourceIDs: map[string]string{"fake-compiler": "external-icarus-id"}, + } + require.NoError(t, svc.AddGame(game)) + + oldSource, oldProfile, oldAll, oldDryRun, oldForce := updateSource, updateProfile, updateAll, updateDryRun, updateForce + oldVerbose, oldNoColor, oldNoHooks := verbose, noColor, noHooks + updateSource = "fake-compiler" + updateProfile = "" + updateAll = false + updateDryRun = false + updateForce = false + verbose = false + noColor = true + noHooks = false + t.Cleanup(func() { + updateSource, updateProfile, updateAll, updateDryRun, updateForce = oldSource, oldProfile, oldAll, oldDryRun, oldForce + verbose, noColor, noHooks = oldVerbose, oldNoColor, oldNoHooks + }) + + const modID, version, fileID = "bear-mount", "3.3", "exmodz-file-id" + gameCache := svc.GetGameCache(game) + require.NoError(t, gameCache.Store(game.ID, "fake-compiler", modID, version, "Bear_Mount_P.pak", []byte("stale-compiled-bytes"))) + require.NoError(t, gameCache.Store(game.ID, "fake-compiler", modID, version, cache.RetainedSourceName(fileID), []byte("retained-exmodz-bytes"))) + versionDir := gameCache.ModPath(game.ID, "fake-compiler", modID, version) + require.NoError(t, cache.MarkFileCompleteWithMembers(versionDir, fileID, []string{"Bear_Mount_P.pak"})) + require.NoError(t, cache.MarkBaseIndexHash(versionDir, fileID, "0000000000000000000000000000000000dead")) + + im := &domain.InstalledMod{ + Mod: domain.Mod{ID: modID, SourceID: "fake-compiler", Name: "Bear Mount", Version: version, GameID: game.ID}, + ProfileName: "default", + UpdatePolicy: domain.UpdateNotify, + Enabled: true, + Deployed: true, + LinkMethod: domain.LinkCopy, + FileIDs: []string{fileID}, + } + require.NoError(t, svc.SaveInstalledMod(im)) + installer := svc.GetInstaller(game) + require.NoError(t, installer.Install(context.Background(), game, &im.Mod, "default")) + + pm := svc.NewProfileManager() + _, cerr := pm.Create(game.ID, "default") + require.NoError(t, cerr) + require.NoError(t, pm.UpsertMod(game.ID, "default", domain.ModReference{SourceID: "fake-compiler", ModID: modID, Version: version, FileIDs: []string{fileID}})) + + return svc, game, compiler, filepath.Join(game.ModPath, "Bear_Mount_P.pak") +} + +// TestDoUpdate_JSON_ReportsRecompileNeeded proves the bulk --json contract +// gained the additive recompile_needed/reason fields (#196) without +// disturbing the rest of the row shape. +func TestDoUpdate_JSON_ReportsRecompileNeeded(t *testing.T) { + svc, game, _, _ := setupDoUpdateRecompileTest(t) + jsonOutput = true + t.Cleanup(func() { jsonOutput = false }) + + var buf bytes.Buffer + oldStdout := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + err := doUpdate(context.Background(), svc, game, nil) + w.Close() + os.Stdout = oldStdout + require.NoError(t, err) + _, _ = buf.ReadFrom(r) + + var out updateJSONOutput + require.NoError(t, json.Unmarshal(buf.Bytes(), &out)) + require.Len(t, out.Updates, 1) + row := out.Updates[0] + assert.Equal(t, "bear-mount", row.ModID) + assert.Equal(t, "3.3", row.Current) + assert.Equal(t, "3.3", row.Available, "a staleness row's available version equals current - the mod hasn't changed") + assert.True(t, row.RecompileNeeded) + assert.Equal(t, "stale_compile", row.Reason) +} + +// TestApplySingleUpdate_Recompile_AppliesAndRedeploys drives `lmm update +// bear-mount` end to end: the stale compile is recompiled from its retained +// source and redeployed, proven via the on-disk (LinkCopy) deployed bytes. +func TestApplySingleUpdate_Recompile_AppliesAndRedeploys(t *testing.T) { + svc, game, compiler, deployedPath := setupDoUpdateRecompileTest(t) + + mod, err := svc.GetInstalledMod("fake-compiler", "bear-mount", "icarus", "default") + require.NoError(t, err) + + err = applySingleUpdate(context.Background(), svc, game, mod, "default") + require.NoError(t, err) + assert.Equal(t, 1, compiler.compileCalls) + + data, err := os.ReadFile(deployedPath) + require.NoError(t, err) + assert.Equal(t, "retained-exmodz-bytes", string(data), "redeploy must reflect the freshly recompiled bytes") +} + +// TestApplySingleUpdate_Recompile_JSON proves the single-mod --json contract +// reports "recompiled" for an applied staleness row, with ToVersion equal +// to FromVersion. +func TestApplySingleUpdate_Recompile_JSON(t *testing.T) { + svc, game, _, _ := setupDoUpdateRecompileTest(t) + jsonOutput = true + t.Cleanup(func() { jsonOutput = false }) + + mod, err := svc.GetInstalledMod("fake-compiler", "bear-mount", "icarus", "default") + require.NoError(t, err) + + var buf bytes.Buffer + oldStdout := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + err = applySingleUpdate(context.Background(), svc, game, mod, "default") + w.Close() + os.Stdout = oldStdout + require.NoError(t, err) + _, _ = buf.ReadFrom(r) + + var out singleUpdateJSON + require.NoError(t, json.Unmarshal(buf.Bytes(), &out)) + assert.Equal(t, "recompiled", out.Status) + assert.Equal(t, "3.3", out.FromVersion) + assert.Equal(t, "3.3", out.ToVersion) +} + +// TestApplySingleUpdate_Recompile_LockedRefuses proves the CLI's locked- +// refusal wording fires for a staleness row too, and never touches the +// locked mod's deployed files. +func TestApplySingleUpdate_Recompile_LockedRefuses(t *testing.T) { + svc, game, compiler, deployedPath := setupDoUpdateRecompileTest(t) + + pm := svc.NewProfileManager() + require.NoError(t, pm.SetModLock(game.ID, "default", "fake-compiler", "bear-mount", "")) + + before, err := os.ReadFile(deployedPath) + require.NoError(t, err) + + mod, err := svc.GetInstalledMod("fake-compiler", "bear-mount", "icarus", "default") + require.NoError(t, err) + + err = applySingleUpdate(context.Background(), svc, game, mod, "default") + require.NoError(t, err, "a locked skip is reported, not returned as an error") + assert.Equal(t, 0, compiler.compileCalls, "a locked mod must never be recompiled") + + after, err := os.ReadFile(deployedPath) + require.NoError(t, err) + assert.Equal(t, before, after, "a locked mod's deployed files must never be touched") +} diff --git a/docs/man/man1/lmm-update.1 b/docs/man/man1/lmm-update.1 index 2444912..ecda796 100644 --- a/docs/man/man1/lmm-update.1 +++ b/docs/man/man1/lmm-update.1 @@ -27,6 +27,12 @@ If the update check itself fails partway through (e.g. a source outage), whatever was learned before the failure is still printed and the command exits non-zero rather than silently claiming success. +.PP +For a compile-deploy game (e.g. Icarus), a compiled mod whose game base pak +has changed since it was last compiled ("recompile needed") is checked and +applied through this same command: no new version, just a same-version +recompile against the current base pak. + .PP --json prints exactly one JSON document to stdout, in one of two shapes: - Bulk check (no mod ID): {game_id, profile, updates: [...], skipped: @@ -34,12 +40,15 @@ exits non-zero rather than silently claiming success. failed partway through; updates/skipped still reflect whatever was learned first. A locked mod's updates[] entry carries "locked": true (omitted when unlocked): the update is reported but will not be - applied until the lock moves or clears. + applied until the lock moves or clears. A recompile-needed entry + carries "recompile_needed": true and "reason": "stale_compile" + (available_version equals current_version - the mod hasn't changed). - Single mod (a mod ID given) or 'update rollback': {mod_id, name, from_version, to_version, changelog, status, reason}. status is one - of "updated", "up_to_date", "skipped", "available" (--dry-run), or - "rolled_back"; reason is set only when status is "skipped" ("pinned", - "local", or "locked"). + of "updated", "up_to_date", "skipped", "available" (--dry-run), + "rolled_back", "recompiled", or "recompile_available" (--dry-run, + same-version base-pak recompile); reason is set only when status is + "skipped" ("pinned", "local", or "locked"). .PP Examples: From 75bffeb9a664ae0d01eee628a233d96149a9602b Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 18:51:59 -0400 Subject: [PATCH 60/96] feat: lmm verify warns on stale compiles (#196) For a DeployCompile game, verify now compares each compiled mod's recorded base-pak fingerprint against the game's live base pak - a new, entirely local/offline pre-pass alongside the existing file-count and version-record checks, applying to every compiled mod regardless of source (including local imports, unlike the version-record check). A mismatch prints "? NAME - RECOMPILE NEEDED (base pak updated - run 'lmm update' to fix)" and --json reports status "stale_compile", counted as a warning (self-healing via `lmm update`, not corruption --fix should repair). make man regenerated. --- cmd/lmm/verify.go | 59 ++++++++++++++++++++++-- cmd/lmm/verify_recompile_test.go | 78 ++++++++++++++++++++++++++++++++ docs/man/man1/lmm-verify.1 | 25 ++++++++-- 3 files changed, 155 insertions(+), 7 deletions(-) create mode 100644 cmd/lmm/verify_recompile_test.go diff --git a/cmd/lmm/verify.go b/cmd/lmm/verify.go index 256ab5a..06a3862 100644 --- a/cmd/lmm/verify.go +++ b/cmd/lmm/verify.go @@ -33,7 +33,7 @@ type verifyFileJSON struct { ModID string `json:"mod_id"` ModName string `json:"mod_name"` FileID string `json:"file_id"` - Status string `json:"status"` // ok, missing, no_checksum, file_count_mismatch, skipped, version_mismatch, version_unverifiable + Status string `json:"status"` // ok, missing, no_checksum, file_count_mismatch, skipped, version_mismatch, version_unverifiable, stale_compile Note string `json:"note,omitempty"` // optional detail: a blocked cache rename, sibling-repair results, a --fix repair/redownload failure reason, or a file-count-check lookup failure - omitted when there's nothing extra to add } @@ -74,6 +74,20 @@ the version of the file that was actually downloaded and deployed): are listed by the source anymore (reinstall or 'lmm update') +For a compile-deploy game (e.g. Icarus), verify also compares each +compiled mod's recorded base-pak fingerprint against the game's live base +pak (#196, "the Friday problem" - a weekly base pak refresh silently +reverts a compiled mod's patched tables, with nothing to notice +otherwise): + + ? NAME - RECOMPILE NEEDED the game's base pak has changed + since this mod was compiled; run + 'lmm update' to recompile it + +This check is entirely local (no source contacted) and applies to every +compiled mod regardless of source, including local imports. --fix does +not repair it - use 'lmm update' (or 'lmm update --all'). + Mods installed from a local source, mods requiring manual download, and mods with no recorded file IDs are skipped silently - there is nothing to check against. If the source can't be reached, the mod is reported @@ -114,9 +128,10 @@ OK case - this is never counted in issues or warnings. --json emits {game_id, profile, files: [{mod_id, mod_name, file_id, status, note}], issues, warnings}; status is one of "ok", "missing", -"no_checksum", "file_count_mismatch", "skipped", "version_mismatch", or -"version_unverifiable"; note adds detail where there's something extra to -say - a blocked cache rename, sibling-repair results, a --fix repair or +"no_checksum", "file_count_mismatch", "skipped", "version_mismatch", +"version_unverifiable", or "stale_compile"; note adds detail where +there's something extra to say - a blocked cache rename, sibling-repair +results, a --fix repair or redownload failure's reason, why a successful re-download stored no checksum, a file-count-check lookup failure, a --fix refusal on a locked record ("locked"), or a locked record's pending convergence detail - and @@ -290,6 +305,42 @@ func doVerify(cmd *cobra.Command, svc *core.Service, game *domain.Game, args []s if err != nil { return fmt.Errorf("getting installed mods: %w", err) } + + // Base-pak staleness check (#196): for a DeployCompile game, compare + // each compiled mod's recorded base-pak fingerprint against the game's + // live base pak. Entirely local/offline - unlike the version-record + // check below, this is NOT skipped for local-source or manual-download + // mods (there is no source dependency at all; see + // Service.CheckBaseStaleness's own doc comment). + if game.DeployMode == domain.DeployCompile { + staleCheckSet := installedMods + if modFilter != "" { + staleCheckSet = nil + for i := range installedMods { + if installedMods[i].ID == modFilter { + staleCheckSet = append(staleCheckSet, installedMods[i]) + } + } + } + stale, serr := svc.CheckBaseStaleness(game, staleCheckSet) + if serr != nil { + if jsonOutput { + jsonFiles = append(jsonFiles, verifyFileJSON{Status: "skipped", Note: fmt.Sprintf("could not check base pak staleness: %v", serr)}) + } else { + fmt.Printf("%s could not check base pak staleness: %v\n", colorYellow("?"), serr) + } + warnings++ + } + checked += len(staleCheckSet) + for _, u := range stale { + if jsonOutput { + jsonFiles = append(jsonFiles, verifyFileJSON{ModID: u.InstalledMod.ID, ModName: u.InstalledMod.Name, Status: "stale_compile"}) + } else { + fmt.Printf("%s %s - RECOMPILE NEEDED (base pak updated - run 'lmm update' to fix)\n", colorYellow("?"), u.InstalledMod.Name) + } + warnings++ + } + } for i := range installedMods { mod := &installedMods[i] if modFilter != "" && mod.ID != modFilter { diff --git a/cmd/lmm/verify_recompile_test.go b/cmd/lmm/verify_recompile_test.go new file mode 100644 index 0000000..16098d6 --- /dev/null +++ b/cmd/lmm/verify_recompile_test.go @@ -0,0 +1,78 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "os" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestDoVerify_StaleCompile_ReportedAsWarning proves `lmm verify` surfaces a +// #196 base-pak staleness row: text mode prints "RECOMPILE NEEDED" and +// --json reports status "stale_compile", counted as a warning (not an +// issue - it's self-healing via `lmm update`, not corruption). +func TestDoVerify_StaleCompile_ReportedAsWarning(t *testing.T) { + svc, game, _, _ := setupDoUpdateRecompileTest(t) + require.NoError(t, svc.SaveFileChecksum("fake-compiler", "bear-mount", game.ID, "default", "exmodz-file-id", "deadbeef")) + + verifyProfile = "default" + t.Cleanup(func() { verifyProfile = "" }) + + cmd := &cobra.Command{} + cmd.SetContext(context.Background()) + + var buf bytes.Buffer + oldStdout := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + err := doVerify(cmd, svc, game, nil) + w.Close() + os.Stdout = oldStdout + require.NoError(t, err) + _, _ = buf.ReadFrom(r) + + output := buf.String() + assert.Contains(t, output, "RECOMPILE NEEDED") + assert.Contains(t, output, "Bear Mount") +} + +// TestDoVerify_StaleCompile_JSON is the --json sibling of the above. +func TestDoVerify_StaleCompile_JSON(t *testing.T) { + svc, game, _, _ := setupDoUpdateRecompileTest(t) + require.NoError(t, svc.SaveFileChecksum("fake-compiler", "bear-mount", game.ID, "default", "exmodz-file-id", "deadbeef")) + + verifyProfile = "default" + jsonOutput = true + t.Cleanup(func() { verifyProfile = ""; jsonOutput = false }) + + cmd := &cobra.Command{} + cmd.SetContext(context.Background()) + + var buf bytes.Buffer + oldStdout := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + err := doVerify(cmd, svc, game, nil) + w.Close() + os.Stdout = oldStdout + require.NoError(t, err) + _, _ = buf.ReadFrom(r) + + var out verifyJSONOutput + require.NoError(t, json.Unmarshal(buf.Bytes(), &out)) + + var found *verifyFileJSON + for i := range out.Files { + if out.Files[i].Status == "stale_compile" { + found = &out.Files[i] + } + } + require.NotNil(t, found, "expected a stale_compile row") + assert.Equal(t, "bear-mount", found.ModID) + assert.GreaterOrEqual(t, out.Warnings, 1) +} diff --git a/docs/man/man1/lmm-verify.1 b/docs/man/man1/lmm-verify.1 index 08a2783..56314b5 100644 --- a/docs/man/man1/lmm-verify.1 +++ b/docs/man/man1/lmm-verify.1 @@ -52,6 +52,24 @@ X NAME - VERSION MISMATCH (recorded X, source reports Y) (reinstall or 'lmm update') .EE +.PP +For a compile-deploy game (e.g. Icarus), verify also compares each +compiled mod's recorded base-pak fingerprint against the game's live base +pak (#196, "the Friday problem" - a weekly base pak refresh silently +reverts a compiled mod's patched tables, with nothing to notice +otherwise): + +.EX +? NAME - RECOMPILE NEEDED the game's base pak has changed + since this mod was compiled; run + 'lmm update' to recompile it +.EE + +.PP +This check is entirely local (no source contacted) and applies to every +compiled mod regardless of source, including local imports. --fix does +not repair it - use 'lmm update' (or 'lmm update --all'). + .PP Mods installed from a local source, mods requiring manual download, and mods with no recorded file IDs are skipped silently - there is nothing @@ -96,9 +114,10 @@ OK case - this is never counted in issues or warnings. .PP --json emits {game_id, profile, files: [{mod_id, mod_name, file_id, status, note}], issues, warnings}; status is one of "ok", "missing", -"no_checksum", "file_count_mismatch", "skipped", "version_mismatch", or -"version_unverifiable"; note adds detail where there's something extra to -say - a blocked cache rename, sibling-repair results, a --fix repair or +"no_checksum", "file_count_mismatch", "skipped", "version_mismatch", +"version_unverifiable", or "stale_compile"; note adds detail where +there's something extra to say - a blocked cache rename, sibling-repair +results, a --fix repair or redownload failure's reason, why a successful re-download stored no checksum, a file-count-check lookup failure, a --fix refusal on a locked record ("locked"), or a locked record's pending convergence detail - and From 7873351ca1a009ad6a76d09a5a7a8390edf66749 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 18:55:20 -0400 Subject: [PATCH 61/96] feat: TUI update screen parity for base-pak recompile (#196) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TUI's update check/apply now go through the same Service.CheckGameUpdates/ApplyRecompile seam the CLI uses: coreProvider.CheckUpdates reports RecompileNeeded rows (new UpdateItem field), and coreProvider.ApplyUpdate dispatches them to Service.ApplyRecompile instead of Service.ApplyUpdate. Rendering: a RecompileNeeded row has FromVersion == ToVersion, so a bare version arrow ("3.3 → 3.3") would read as a no-op. New UpdateItem.VersionLabel() renders "(base pak updated)" for such rows instead, used everywhere an update's version change is shown - the apply-updates modal, its result lines, and the changelog picker/overlay - so all of them read sanely without duplicating the branch four times. --- internal/tui/actions.go | 4 +- internal/tui/actions_provider.go | 20 +++ internal/tui/mutations.go | 4 +- internal/tui/service_core.go | 31 +++- internal/tui/service_core_recompile_test.go | 158 ++++++++++++++++++++ 5 files changed, 206 insertions(+), 11 deletions(-) create mode 100644 internal/tui/service_core_recompile_test.go diff --git a/internal/tui/actions.go b/internal/tui/actions.go index 6552799..458c55a 100644 --- a/internal/tui/actions.go +++ b/internal/tui/actions.go @@ -484,7 +484,7 @@ func (m Model) openChangelogFromUpdateModal() (tea.Model, tea.Cmd) { options := make([]pickerOption, len(updates)) for i, u := range updates { - options[i] = pickerOption{Label: fmt.Sprintf("%s %s → %s", u.Name, u.FromVersion, u.ToVersion)} + options[i] = pickerOption{Label: fmt.Sprintf("%s %s", u.Name, u.VersionLabel())} } m.picker = &pendingPicker{ title: "View changelog", @@ -507,7 +507,7 @@ func (m Model) openChangelogFromUpdateModal() (tea.Model, tea.Cmd) { // (the source reported none) renders the single line "no changelog // available" instead of an empty panel. func changelogOverlay(u UpdateItem) *infoOverlay { - title := fmt.Sprintf("%s %s → %s", u.Name, u.FromVersion, u.ToVersion) + title := fmt.Sprintf("%s %s", u.Name, u.VersionLabel()) lines := []string{"no changelog available"} if u.Changelog != "" { lines = strings.Split(u.Changelog, "\n") diff --git a/internal/tui/actions_provider.go b/internal/tui/actions_provider.go index 5539c9c..c49b3ce 100644 --- a/internal/tui/actions_provider.go +++ b/internal/tui/actions_provider.go @@ -280,6 +280,26 @@ type UpdateItem struct { // is false" contract. Locked bool LockedVersion string + // RecompileNeeded marks a #196 base-pak staleness row: a DeployCompile + // mod whose deployed compile no longer matches the game's live base + // pak. ToVersion equals FromVersion in this case - the mod itself + // hasn't changed, only the base pak has - and ApplyUpdate routes such a + // row to Service.ApplyRecompile instead of Service.ApplyUpdate. + RecompileNeeded bool +} + +// VersionLabel renders u's version change for display: the normal +// " → " arrow for a real update, or "(base pak updated)" for a +// #196 RecompileNeeded row, where FromVersion == ToVersion and an arrow +// would misleadingly read as a no-op. Used everywhere an UpdateItem's +// version change is shown - the apply-updates modal, its result lines, and +// the changelog picker/overlay - so all of them read sanely for a +// staleness row without duplicating this branch four times. +func (u UpdateItem) VersionLabel() string { + if u.RecompileNeeded { + return "(base pak updated)" + } + return fmt.Sprintf("%s → %s", u.FromVersion, u.ToVersion) } // UpdatesView is CheckUpdates' result: the available updates plus any diff --git a/internal/tui/mutations.go b/internal/tui/mutations.go index 00fe1a2..578b48b 100644 --- a/internal/tui/mutations.go +++ b/internal/tui/mutations.go @@ -1447,7 +1447,7 @@ func (m Model) resolveCheckUpdatesFailure(msg checkUpdatesFailedMsg) (Model, tea func updateDetailLines(view UpdatesView) []string { lines := make([]string, 0, len(view.Updates)+1) for _, u := range view.Updates { - line := fmt.Sprintf("%s %s → %s", u.Name, u.FromVersion, u.ToVersion) + line := fmt.Sprintf("%s %s", u.Name, u.VersionLabel()) if u.Locked { line += fmt.Sprintf(" [locked@%s]", u.LockedVersion) } @@ -1552,7 +1552,7 @@ func applyUpdatesSequentially(ctx context.Context, actions ActionProvider, updat } applied++ warnings = append(warnings, outcome.Warnings...) - resultLines = append(resultLines, fmt.Sprintf("✓ %s %s → %s", u.Name, u.FromVersion, u.ToVersion)) + resultLines = append(resultLines, fmt.Sprintf("✓ %s %s", u.Name, u.VersionLabel())) } return ActionOutcome{ Message: fmt.Sprintf("Applied %d update(s)", applied), diff --git a/internal/tui/service_core.go b/internal/tui/service_core.go index 6701621..166a368 100644 --- a/internal/tui/service_core.go +++ b/internal/tui/service_core.go @@ -1423,7 +1423,7 @@ func (p *coreProvider) CheckUpdates(ctx context.Context) (UpdatesView, error) { return UpdatesView{}, fmt.Errorf("loading installed mods for %s/%s: %w", game.ID, profile, err) } - updates, checkErr := p.svc.NewUpdater().CheckUpdates(ctx, game, installed) + updates, checkErr := p.svc.CheckGameUpdates(ctx, game, installed) // #143: join the profile YAML's lock state onto the update rows - the // same projection (and the same nil-safe "an unreadable profile leaves @@ -1446,6 +1446,7 @@ func (p *coreProvider) CheckUpdates(ctx context.Context) (UpdatesView, error) { FromVersion: u.InstalledMod.Version, ToVersion: u.NewVersion, Changelog: core.CleanChangelog(u.Changelog), Locked: isLocked, LockedVersion: lockedVersion, + RecompileNeeded: u.RecompileNeeded, }) } if skipped := updateSkipWarning(core.CountUpdateSkips(installed)); skipped != "" { @@ -1464,12 +1465,16 @@ func (p *coreProvider) CheckUpdates(ctx context.Context) (UpdatesView, error) { // ApplyUpdate applies u with the SAME hook configuration cmd/lmm/update.go's // applyUpdate passes (Force=false, its default). u is re-checked via -// CheckUpdates for just this one mod first - mirroring +// CheckGameUpdates for just this one mod first - mirroring // cmd/lmm/update.go's applySingleUpdate, which does the same before calling // applyUpdate - rather than reconstructing a bare domain.Update from u's own // fields: UpdateItem carries no FileIDReplacements (see its doc comment), // and a real update may need that superseded-file-ID mapping to install -// correctly; only a fresh CheckUpdates call can supply it. +// correctly; only a fresh check call can supply it. +// +// #196: a RecompileNeeded row (NewVersion == the mod's current version - a +// base-pak staleness signal, not a real update) is routed to +// Service.ApplyRecompile instead, which has no hooks/options to configure. func (p *coreProvider) ApplyUpdate(ctx context.Context, u UpdateItem, progress func(ActionProgress)) (ActionOutcome, error) { game := p.currentGame() profile := p.currentProfile() @@ -1478,7 +1483,7 @@ func (p *coreProvider) ApplyUpdate(ctx context.Context, u UpdateItem, progress f return ActionOutcome{}, fmt.Errorf("getting installed mod %s: %w", u.Name, err) } - updates, err := p.svc.NewUpdater().CheckUpdates(ctx, game, []domain.InstalledMod{*mod}) + updates, err := p.svc.CheckGameUpdates(ctx, game, []domain.InstalledMod{*mod}) if err != nil { return ActionOutcome{}, mapUpdateNetworkError(fmt.Sprintf("checking update for %s", u.Name), u.Source, err) } @@ -1487,6 +1492,21 @@ func (p *coreProvider) ApplyUpdate(ctx context.Context, u UpdateItem, progress f } upd := updates[0] + adapter := deployProgressAdapter(progress, func(p core.DeployProgress) (ActionProgress, bool) { + return updateProgressLine(u.Name, p) + }) + + if upd.RecompileNeeded { + result, err := p.svc.ApplyRecompile(ctx, game, profile, upd.InstalledMod, adapter) + if err != nil { + return ActionOutcome{}, mapUpdateNetworkError(fmt.Sprintf("recompiling %s", u.Name), u.Source, err) + } + return ActionOutcome{ + Message: fmt.Sprintf("Recompiled %q (base pak updated)", u.Name), + Warnings: mergeDiagnostics(result.Warnings, result.Notes), + }, nil + } + opts := core.UpdateOptions{ Hooks: p.resolvedHooks(game, profile), HookRunner: p.hookRunner(), @@ -1494,9 +1514,6 @@ func (p *coreProvider) ApplyUpdate(ctx context.Context, u UpdateItem, progress f Force: false, } - adapter := deployProgressAdapter(progress, func(p core.DeployProgress) (ActionProgress, bool) { - return updateProgressLine(u.Name, p) - }) result, err := p.svc.ApplyUpdate(ctx, game, profile, upd, opts, adapter) if err != nil { return ActionOutcome{}, mapUpdateNetworkError(fmt.Sprintf("updating %s", u.Name), u.Source, err) diff --git a/internal/tui/service_core_recompile_test.go b/internal/tui/service_core_recompile_test.go new file mode 100644 index 0000000..e0e4c7a --- /dev/null +++ b/internal/tui/service_core_recompile_test.go @@ -0,0 +1,158 @@ +package tui_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/core" + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/source" + "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" + "github.com/DonovanMods/linux-mod-manager/internal/tui" + "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" + "github.com/stretchr/testify/require" +) + +// recompileFakeSource is a minimal ModSource + source.Compiler standing in +// for internal/source/icarus.Icarus, mirroring +// internal/core/service_icarus_compile_test.go's fakeCompilerSource at the +// TUI layer. +type recompileFakeSource struct { + compileCalls int +} + +func (s *recompileFakeSource) ID() string { return "fake-compiler" } +func (s *recompileFakeSource) Name() string { return "Fake Compiler Source" } +func (s *recompileFakeSource) AuthURL() string { return "" } +func (s *recompileFakeSource) ExchangeToken(ctx context.Context, code string) (*source.Token, error) { + return nil, source.ErrNotSupported +} +func (s *recompileFakeSource) Search(ctx context.Context, query source.SearchQuery) (source.SearchResult, error) { + return source.SearchResult{}, source.ErrNotSupported +} +func (s *recompileFakeSource) GetMod(ctx context.Context, gameID, modID string) (*domain.Mod, error) { + return nil, source.ErrNotSupported +} +func (s *recompileFakeSource) GetDependencies(ctx context.Context, mod *domain.Mod) ([]domain.ModReference, error) { + return nil, source.ErrNotSupported +} +func (s *recompileFakeSource) GetModFiles(ctx context.Context, mod *domain.Mod) ([]domain.DownloadableFile, error) { + return nil, source.ErrNotSupported +} +func (s *recompileFakeSource) GetDownloadURL(ctx context.Context, mod *domain.Mod, fileID string) (string, error) { + return "", source.ErrNotSupported +} +func (s *recompileFakeSource) CheckUpdates(ctx context.Context, installed []domain.InstalledMod) ([]domain.Update, error) { + return nil, nil +} +func (s *recompileFakeSource) Compile(ctx context.Context, basePakPath, sourceFilePath, outputPath string) error { + s.compileCalls++ + data, err := os.ReadFile(sourceFilePath) + if err != nil { + return err + } + return os.WriteFile(outputPath, data, 0o644) +} + +var ( + _ source.ModSource = (*recompileFakeSource)(nil) + _ source.Compiler = (*recompileFakeSource)(nil) +) + +// newRecompileActionsFixture builds a DeployCompile game with an installed, +// deployed compiled mod whose recorded base-pak fingerprint is wrong, and +// returns the ActionProvider (#196's CLI/TUI parity seam), the fake +// compiler, and the deployed file's game-dir path. +func newRecompileActionsFixture(t *testing.T) (tui.ActionProvider, *recompileFakeSource, string) { + t.Helper() + + installDir := t.TempDir() + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + w, err := unrealpak.Create(basePak) + require.NoError(t, err) + require.NoError(t, w.AddFile("Data/D_Fixture.json", []byte(`{"fixture":true}`))) + require.NoError(t, w.Close()) + + svc, err := core.NewService(core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + compiler := &recompileFakeSource{} + svc.RegisterSource(compiler) + + game := &domain.Game{ + ID: "icarus", Name: "Icarus", InstallPath: installDir, ModPath: t.TempDir(), + DeployMode: domain.DeployCompile, LinkMethod: domain.LinkCopy, + SourceIDs: map[string]string{"fake-compiler": "external-icarus-id"}, + } + require.NoError(t, svc.AddGame(game)) + + pm := svc.NewProfileManager() + _, err = pm.Create(game.ID, "default") + require.NoError(t, err) + require.NoError(t, pm.SetDefault(game.ID, "default")) + + const modID, version, fileID = "bear-mount", "3.3", "exmodz-file-id" + gameCache := svc.GetGameCache(game) + require.NoError(t, gameCache.Store(game.ID, "fake-compiler", modID, version, "Bear_Mount_P.pak", []byte("stale-compiled-bytes"))) + require.NoError(t, gameCache.Store(game.ID, "fake-compiler", modID, version, cache.RetainedSourceName(fileID), []byte("retained-exmodz-bytes"))) + versionDir := gameCache.ModPath(game.ID, "fake-compiler", modID, version) + require.NoError(t, cache.MarkFileCompleteWithMembers(versionDir, fileID, []string{"Bear_Mount_P.pak"})) + require.NoError(t, cache.MarkBaseIndexHash(versionDir, fileID, "0000000000000000000000000000000000dead")) + + im := &domain.InstalledMod{ + Mod: domain.Mod{ID: modID, SourceID: "fake-compiler", Name: "Bear Mount", Version: version, GameID: game.ID}, + ProfileName: "default", + UpdatePolicy: domain.UpdateNotify, + Enabled: true, + Deployed: true, + LinkMethod: domain.LinkCopy, + FileIDs: []string{fileID}, + } + require.NoError(t, svc.SaveInstalledMod(im)) + installer := svc.GetInstaller(game) + require.NoError(t, installer.Install(context.Background(), game, &im.Mod, "default")) + require.NoError(t, pm.UpsertMod(game.ID, "default", domain.ModReference{SourceID: "fake-compiler", ModID: modID, Version: version, FileIDs: []string{fileID}})) + + return tui.NewCoreActions(svc, game, "default"), compiler, filepath.Join(game.ModPath, "Bear_Mount_P.pak") +} + +// TestCoreProviderActions_CheckUpdates_ReportsRecompileNeeded proves the TUI +// update check reports a #196 base-pak staleness row through the same +// CheckGameUpdates seam the CLI uses. +func TestCoreProviderActions_CheckUpdates_ReportsRecompileNeeded(t *testing.T) { + actions, _, _ := newRecompileActionsFixture(t) + + view, err := actions.CheckUpdates(context.Background()) + require.NoError(t, err) + require.Len(t, view.Updates, 1) + u := view.Updates[0] + require.Equal(t, "bear-mount", u.ID) + require.True(t, u.RecompileNeeded) + require.Equal(t, u.FromVersion, u.ToVersion, "a staleness row has no real version change") + require.Equal(t, "(base pak updated)", u.VersionLabel()) +} + +// TestCoreProviderActions_ApplyUpdate_Recompile_AppliesAndRedeploys proves +// ApplyUpdate dispatches a RecompileNeeded row to Service.ApplyRecompile: +// the retained source is recompiled and redeployed, provable via the +// on-disk (LinkCopy) deployed bytes. +func TestCoreProviderActions_ApplyUpdate_Recompile_AppliesAndRedeploys(t *testing.T) { + actions, compiler, deployedPath := newRecompileActionsFixture(t) + + view, err := actions.CheckUpdates(context.Background()) + require.NoError(t, err) + require.Len(t, view.Updates, 1) + + outcome, err := actions.ApplyUpdate(context.Background(), view.Updates[0], nil) + require.NoError(t, err) + require.Equal(t, 1, compiler.compileCalls) + require.Contains(t, outcome.Message, "Recompiled") + + data, err := os.ReadFile(deployedPath) + require.NoError(t, err) + require.Equal(t, "retained-exmodz-bytes", string(data), "redeploy must reflect the freshly recompiled bytes") +} From f33257ee19a6d5ddbcdda5b111403353af9fa385 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 18:56:27 -0400 Subject: [PATCH 62/96] docs: CHANGELOG entry for base-pak recompile (#196); trunk errcheck fix Also fixes an errcheck lint hit (unchecked os.Pipe writer Close in two new test files) trunk flagged after the TUI-parity commit. --- CHANGELOG.md | 1 + cmd/lmm/update_recompile_test.go | 4 ++-- cmd/lmm/verify_recompile_test.go | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c5babd..b2f31b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Icarus built-in mod source** (`internal/source/icarus`): a public, unauthenticated Firestore-backed catalog (Project Daedalus) — `lmm search`/`install`/`update` work against it like NexusMods/CurseForge. A `.exmodz` mod file now compiles into a deployable `_P.pak` at download time via a new, game-agnostic `internal/unrealpak` PAK reader/writer and the new `deploy_mode: compile` game setting; a plain `.pak` file from the same catalog is unaffected and deploys through the existing extract/copy pipeline unchanged. Base data tables are read directly from the installed game's own `data.pak`, so a compile always matches the installed game version and works entirely offline; `internal/unrealpak` reads both the stored and the Zlib-compressed entries that pak contains, using only the standard library (#136, #175) - `lmm game detect` now recognizes Icarus (Steam App ID `1149460`) and generates a complete `games.yaml` entry for it (`deploy_mode: compile`, `sources: {icarus: icarus}`) — no more hand-editing `games.yaml` to get started. The known-games schema (`steam-games.yaml`, built-in or your own override) gained two optional fields, `deploy_mode` and `sources`, generalizing detection beyond NexusMods-only games; every existing entry is unaffected (#177) - Custom `api` sources' `search` endpoint gains `{category}`/`{tags}` path placeholders, fed from `SearchQuery.Category`/`.Tags` (URL-escaped; multiple tags comma-joined) — previously these were silently dropped with no way for a declarative source to express category/tag filtering. A definition whose `search` path omits the new placeholders is unaffected: the values are computed but never substituted in, matching today's behavior exactly (#120) +- Compiled mods (`deploy_mode: compile`, e.g. Icarus) now recover automatically when the game's base `data.pak` changes underneath them — "the Friday problem": a weekly base-pak refresh used to silently revert a compiled mod's patched tables, with nothing to notice. Compiling now records the base pak's footer fingerprint and retains a copy of the original `.exmodz` beside the compiled `_P.pak`, both invisible to deployment. `lmm update` (CLI and TUI) checks every compiled mod's fingerprint against the game's current base pak and reports a same-version "recompile needed" row (additive `--json` field `recompile_needed`/`reason`) alongside normal version updates; applying it recompiles in place from the retained `.exmodz` (falling back to a re-download when possible) and redeploys — pinned mods recompile normally, locked mods are refused with the same loud lock warning a real update gets. Pre-existing compiled installs without the new fingerprint are left alone rather than guessed at (indistinguishable from a plain prebuilt `.pak`); they pick up fingerprinting on their next real recompile. `lmm verify` gains a matching "RECOMPILE NEEDED" warning row (`stale_compile`) (#196) ### Changed diff --git a/cmd/lmm/update_recompile_test.go b/cmd/lmm/update_recompile_test.go index 3ffc839..29018a5 100644 --- a/cmd/lmm/update_recompile_test.go +++ b/cmd/lmm/update_recompile_test.go @@ -104,7 +104,7 @@ func TestDoUpdate_JSON_ReportsRecompileNeeded(t *testing.T) { r, w, _ := os.Pipe() os.Stdout = w err := doUpdate(context.Background(), svc, game, nil) - w.Close() + _ = w.Close() os.Stdout = oldStdout require.NoError(t, err) _, _ = buf.ReadFrom(r) @@ -154,7 +154,7 @@ func TestApplySingleUpdate_Recompile_JSON(t *testing.T) { r, w, _ := os.Pipe() os.Stdout = w err = applySingleUpdate(context.Background(), svc, game, mod, "default") - w.Close() + _ = w.Close() os.Stdout = oldStdout require.NoError(t, err) _, _ = buf.ReadFrom(r) diff --git a/cmd/lmm/verify_recompile_test.go b/cmd/lmm/verify_recompile_test.go index 16098d6..e711a38 100644 --- a/cmd/lmm/verify_recompile_test.go +++ b/cmd/lmm/verify_recompile_test.go @@ -31,7 +31,7 @@ func TestDoVerify_StaleCompile_ReportedAsWarning(t *testing.T) { r, w, _ := os.Pipe() os.Stdout = w err := doVerify(cmd, svc, game, nil) - w.Close() + _ = w.Close() os.Stdout = oldStdout require.NoError(t, err) _, _ = buf.ReadFrom(r) @@ -58,7 +58,7 @@ func TestDoVerify_StaleCompile_JSON(t *testing.T) { r, w, _ := os.Pipe() os.Stdout = w err := doVerify(cmd, svc, game, nil) - w.Close() + _ = w.Close() os.Stdout = oldStdout require.NoError(t, err) _, _ = buf.ReadFrom(r) From 74d446f2e0e2efbff850065cda2ee80399f91afd Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 19:08:35 -0400 Subject: [PATCH 63/96] fix: stat-error handling and filename sanitization in recompile (#196 review) - ApplyRecompile's retained-source os.Stat check treated ANY error as "missing" (masking permission/I/O problems behind misleading redownload/re-import text). New ClassifyRetainedSourceStatError pure classifier: only errors.Is(err, fs.ErrNotExist) means missing; any other stat error surfaces as its own actionable, wrapped failure. Unit-tested directly (a real unreadable-parent fixture would also break ApplyRecompile's own prepareStaging seed step, which reads the same version directory earlier in the same call). - The redownload fallback's dlPath joined a source-controlled DownloadableFile.FileName verbatim into the staging path (filepath.Join(tempDir, match.FileName)), letting a malicious/buggy source traverse outside tempDir (e.g. "../evil.exmodz"). Now sanitized via filepath.Base, matching this package's existing convention for the same concern (importer.go's filepath.Base(archivePath), service.go's filepath.Base(localPath) fallback). DownloadModToCache's archivePath (service.go:514) has the identical unsanitized filepath.Join(tempDir, file.FileName) pattern and is therefore equally vulnerable, but is out of scope for this review wave targeting updater.go - noted for a follow-up. --- .../service_apply_recompile_review_test.go | 165 ++++++++++++++++++ internal/core/updater.go | 39 ++++- 2 files changed, 202 insertions(+), 2 deletions(-) create mode 100644 internal/core/service_apply_recompile_review_test.go diff --git a/internal/core/service_apply_recompile_review_test.go b/internal/core/service_apply_recompile_review_test.go new file mode 100644 index 0000000..5ee350c --- /dev/null +++ b/internal/core/service_apply_recompile_review_test.go @@ -0,0 +1,165 @@ +package core_test + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/core" + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" + "github.com/stretchr/testify/require" +) + +// TestApplyRecompile_RetainedSourceStatError_SurfacesActionably pins the +// #196 review finding: os.Stat's error on the retained source path was +// treated as "missing" unconditionally, masking a genuine permission/I/O +// problem behind misleading "retained source is missing" / redownload- +// fallback text. A non-ENOENT stat error must surface as its own +// actionable failure instead. +// +// A real unreadable-parent fixture would ALSO break ApplyRecompile's own +// prepareStaging seed step (which reads the same version directory before +// this check ever runs), so this exercises the extracted pure classifier +// directly - core.ClassifyRetainedSourceStatError - rather than trying to +// force a filesystem-level permission error through the full call. +func TestClassifyRetainedSourceStatError(t *testing.T) { + t.Run("nil error: present", func(t *testing.T) { + missing, err := core.ClassifyRetainedSourceStatError(nil) + require.False(t, missing) + require.NoError(t, err) + }) + + t.Run("not-exist: missing, no error", func(t *testing.T) { + notExist := &os.PathError{Op: "stat", Path: "/x", Err: os.ErrNotExist} + missing, err := core.ClassifyRetainedSourceStatError(notExist) + require.True(t, missing) + require.NoError(t, err) + }) + + t.Run("permission denied: not missing, actionable error", func(t *testing.T) { + permErr := &os.PathError{Op: "stat", Path: "/x", Err: os.ErrPermission} + missing, err := core.ClassifyRetainedSourceStatError(permErr) + require.False(t, missing, "a permission error must never be folded into 'missing'") + require.Error(t, err) + require.ErrorIs(t, err, permErr) + }) +} + +// TestApplyRecompile_RetainedSourceStatError_Integration proves the +// classifier is actually wired into ApplyRecompile: a stat error that is +// NOT "not exist" must abort with an actionable error, WITHOUT ever +// treating the entry as eligible for the local-mod-fails-loud or +// redownload-fallback text (which would misrepresent a permission/I/O +// problem as "the file is gone"). +// +// Simulated here by replacing the retained source with a directory (a +// real, portable way to make a SECOND os.Stat-adjacent operation fail +// without touching filesystem permissions): os.Stat itself still succeeds +// on a directory, so this proves the narrower regression - that a +// genuinely present (if wrong-shaped) entry is never silently redownloaded +// - while the classifier unit tests above cover the permission-error path +// directly. +func TestApplyRecompile_RetainedSourceIsDirectory_DoesNotSilentlyRedownload(t *testing.T) { + fx := seedCompiledInstalledMod(t, domain.LinkCopy, "fake-compiler", "0000000000000000000000000000000000dead") + + gameCache := fx.svc.GetGameCache(fx.game) + retainedPath := gameCache.GetFilePath(fx.game.ID, "fake-compiler", "bear-mount", "3.3", cache.RetainedSourceName("exmodz-file-id")) + require.NoError(t, os.Remove(retainedPath)) + require.NoError(t, os.Mkdir(retainedPath, 0o755)) + + compiler := &redownloadCompilerSource{ + fakeCompilerSource: &fakeCompilerSource{}, + downloadBody: "should-never-be-fetched", + files: []domain.DownloadableFile{{ID: "exmodz-file-id", FileName: "Bear_Mount.exmodz"}}, + } + compiler.start(t) + fx.svc.RegisterSource(compiler) + + mod, err := fx.svc.GetInstalledMod("fake-compiler", "bear-mount", "icarus", "default") + require.NoError(t, err) + + _, err = fx.svc.ApplyRecompile(context.Background(), fx.game, "default", *mod, nil) + require.Error(t, err, "a present-but-unusable retained source must fail loud, not silently redownload or succeed") +} + +// TestApplyRecompile_RedownloadedFileName_SanitizedAgainstTraversal pins +// the #196 review finding: a source-controlled DownloadableFile.FileName +// joined verbatim into the staging path can traverse outside the intended +// staging directory (e.g. "../evil.exmodz"). The fix must sanitize via +// filepath.Base before joining, mirroring the existing convention used +// elsewhere in this package for exactly this concern (importer.go's +// filepath.Base(archivePath), service.go's filepath.Base(localPath) +// fallback). +func TestApplyRecompile_RedownloadedFileName_SanitizedAgainstTraversal(t *testing.T) { + dataDir := t.TempDir() + installDir := t.TempDir() + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + writeFakeBasePak(t, basePak) + + svc, err := core.NewService(core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: dataDir, CacheDir: t.TempDir()}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + game := &domain.Game{ + ID: "icarus", InstallPath: installDir, ModPath: t.TempDir(), + DeployMode: domain.DeployCompile, LinkMethod: domain.LinkCopy, + SourceIDs: map[string]string{"fake-compiler": "external-icarus-id"}, + } + require.NoError(t, svc.AddGame(game)) + + const modID, version, fileID = "bear-mount", "3.3", "exmodz-file-id" + gameCache := svc.GetGameCache(game) + require.NoError(t, gameCache.Store(game.ID, "fake-compiler", modID, version, "Bear_Mount_P.pak", []byte("stale-compiled-bytes"))) + versionDir := gameCache.ModPath(game.ID, "fake-compiler", modID, version) + require.NoError(t, cache.MarkFileCompleteWithMembers(versionDir, fileID, []string{"Bear_Mount_P.pak"})) + require.NoError(t, cache.MarkBaseIndexHash(versionDir, fileID, "0000000000000000000000000000000000dead")) + // Deliberately NO retained source stored - forces the redownload path, + // which is where the vulnerable filepath.Join(tempDir, match.FileName) + // lives. + + im := &domain.InstalledMod{ + Mod: domain.Mod{ID: modID, SourceID: "fake-compiler", Name: "Bear Mount", Version: version, GameID: game.ID}, + ProfileName: "default", + UpdatePolicy: domain.UpdateNotify, + Enabled: true, + Deployed: true, + LinkMethod: domain.LinkCopy, + FileIDs: []string{fileID}, + } + require.NoError(t, svc.SaveInstalledMod(im)) + installer := svc.GetInstaller(game) + require.NoError(t, installer.Install(context.Background(), game, &im.Mod, "default")) + pm := svc.NewProfileManager() + _, cerr := pm.Create(game.ID, "default") + require.NoError(t, cerr) + require.NoError(t, pm.UpsertMod(game.ID, "default", domain.ModReference{SourceID: "fake-compiler", ModID: modID, Version: version, FileIDs: []string{fileID}})) + + compiler := &redownloadCompilerSource{ + fakeCompilerSource: &fakeCompilerSource{}, + downloadBody: "redownloaded-exmodz-bytes", + files: []domain.DownloadableFile{{ID: fileID, FileName: "../evil-traversal.exmodz"}}, + } + compiler.start(t) + svc.RegisterSource(compiler) + + mod, err := svc.GetInstalledMod("fake-compiler", "bear-mount", "icarus", "default") + require.NoError(t, err) + + _, err = svc.ApplyRecompile(context.Background(), game, "default", *mod, nil) + require.NoError(t, err) + + // newStagingDir("lmm-recompile-*") creates its scratch dir directly + // under dataDir/downloads (Service.stagingRoot) - an UNSANITIZED + // filepath.Join(tempDir, "../evil-traversal.exmodz") climbs exactly one + // level out of that scratch dir, landing at dataDir/downloads/ + // evil-traversal.exmodz. That parent is never removed (only tempDir + // itself is), so an escaped write would persist right here. + escapedPath := filepath.Join(dataDir, "downloads", "evil-traversal.exmodz") + _, statErr := os.Stat(escapedPath) + require.True(t, errors.Is(statErr, os.ErrNotExist), fmt.Sprintf("a traversal filename must never write outside the staging tempDir (found %s)", escapedPath)) +} diff --git a/internal/core/updater.go b/internal/core/updater.go index 6e1af61..a3753aa 100644 --- a/internal/core/updater.go +++ b/internal/core/updater.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io/fs" "os" "path/filepath" @@ -279,6 +280,27 @@ func (s *Service) CheckGameUpdates(ctx context.Context, game *domain.Game, insta // stale on-disk bytes (a symlink deployment already reflects the new cache // content once the atomic swap above lands, so this step is a correctness // no-op for it, not a wasted one). +// +// ClassifyRetainedSourceStatError interprets os.Stat's error on a retained +// compile source path (#196 review): only a genuine "not exist" means +// missing (ok=true, err=nil) and falls through to the redownload/local- +// fails-loud logic below. Any OTHER stat error - permission denied, an I/O +// error, ... - is NOT "missing": folding it into the same code path would +// misreport a real filesystem problem as "re-import to restore it" or +// silently trigger a redownload the retained file didn't actually warrant. +// Such an error is returned instead, wrapped with %w so callers/tests can +// still errors.Is/As through to the original. Exported so it can be unit +// tested directly (deps/cache.go-style: this is a pure classifier, not a +// filesystem operation). +func ClassifyRetainedSourceStatError(statErr error) (missing bool, err error) { + if statErr == nil { + return false, nil + } + if errors.Is(statErr, fs.ErrNotExist) { + return true, nil + } + return false, statErr +} func (s *Service) ApplyRecompile(ctx context.Context, game *domain.Game, profileName string, mod domain.InstalledMod, progress func(DeployProgress)) (result *UpdateApplyResult, err error) { result = &UpdateApplyResult{} emit := func(p DeployProgress) { @@ -352,7 +374,12 @@ func (s *Service) ApplyRecompile(ctx context.Context, game *domain.Game, profile retainedPath := gameCache.GetFilePath(game.ID, mod.SourceID, mod.ID, mod.Version, cache.RetainedSourceName(fileID)) sourcePath := retainedPath - if _, statErr := os.Stat(retainedPath); statErr != nil { + _, statErr := os.Stat(retainedPath) + missing, statErr := ClassifyRetainedSourceStatError(statErr) + if statErr != nil { + return result, fmt.Errorf("%s: checking retained compile source for %q: %w", mod.Name, fileID, statErr) + } + if missing { if mod.SourceID == domain.SourceLocal { return result, fmt.Errorf("%s: retained compile source for %q is missing and this mod has no remote source to re-download from - re-import the .exmodz to restore it", mod.Name, fileID) } @@ -379,7 +406,15 @@ func (s *Service) ApplyRecompile(ctx context.Context, game *domain.Game, profile return result, terr } defer os.RemoveAll(tempDir) //nolint:errcheck - dlPath := filepath.Join(tempDir, match.FileName) + // filepath.Base: match.FileName is source-controlled (a + // DownloadableFile from mod.SourceID's own listing) and must + // not be trusted as a path component verbatim - an entry like + // "../../evil.exmodz" would otherwise escape tempDir (#196 + // review). Same sanitization idiom already used elsewhere in + // this package for a source-derived name (importer.go's + // filepath.Base(archivePath), service.go's + // filepath.Base(localPath) fallback). + dlPath := filepath.Join(tempDir, filepath.Base(match.FileName)) evt := base evt.Phase, evt.Detail = UpdateNote, fmt.Sprintf("retained compile source missing for %s - re-downloading", destName) emit(evt) From fb3a211a58d4b149de144e67678a75bb095ebb69 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 19:14:41 -0400 Subject: [PATCH 64/96] fix: sanitize source filenames at every temp-path join (#196 review) DownloadModToCache's archivePath (the MAIN download path, not just the recompile fallback) joined a source-controlled DownloadableFile.FileName verbatim into tempDir/stagePath in three places: the download's own archivePath, the DeployCopy destPath, and the DeployCompile branch's compiledFileName-derived destName (compiledFileName only trims/adds a suffix - it does not strip directory components, so a traversal payload in the stem survives through it unless re-sanitized). ingestLocalToCache (the file:// local-ingest path, also fed by a ModSource's declared FileName) had the identical gap. All four now sanitize via filepath.Base, mirroring the convention already used elsewhere in this package (importer.go's filepath.Base(archivePath), service.go's filepath.Base(localPath) fallback) and the redownload-fallback fix already landed in updater.go. Repo-wide sweep for other filepath.Join(, ) sites (grep -rn "filepath.Join(" | grep -i filename/\.name): - cmd/lmm/import.go:695 (r.FileName) - SAFE: ScanResult.FileName always comes from a LOCAL directory walk (os.ReadDir entry.Name()), never a remote source. - internal/core/importer.go, internal/storage/cache/cache.go, internal/source/custom/{directory,metadata/modinfo}.go - all use entry.Name() from a local os.ReadDir/WalkDir, never remote input. - internal/storage/config/profiles.go - profileName/gameID are local CLI-supplied identifiers, a different (non-remote-source) threat class, out of scope for this review. No other remote-source-controlled filename reaches filepath.Join unsanitized after this commit. --- internal/core/service.go | 28 +++-- internal/core/service_download_local_test.go | 32 +++++ .../core/service_download_traversal_test.go | 113 ++++++++++++++++++ 3 files changed, 166 insertions(+), 7 deletions(-) create mode 100644 internal/core/service_download_traversal_test.go diff --git a/internal/core/service.go b/internal/core/service.go index 728f662..111556e 100644 --- a/internal/core/service.go +++ b/internal/core/service.go @@ -510,8 +510,15 @@ func (s *Service) DownloadModToCache(ctx context.Context, gameCache *cache.Cache } }() - // Download the file - archivePath := filepath.Join(tempDir, file.FileName) + // Download the file. safeFileName sanitizes file.FileName - a + // SOURCE-CONTROLLED value (NexusMods/CurseForge/Icarus/a custom source's + // own declared filename) - before it is ever used as a path component: + // an entry like "../../evil" would otherwise let a malicious or buggy + // source escape tempDir/stagePath (#196 review). Used for every + // path-construction use of the filename below; file.FileName itself is + // left untouched for display purposes (the SHA256 mismatch message). + safeFileName := filepath.Base(file.FileName) + archivePath := filepath.Join(tempDir, safeFileName) var headers map[string]string if hp, ok := src.(source.DownloadHeaderProvider); ok { headers = hp.DownloadHeaders(url) @@ -533,7 +540,7 @@ func (s *Service) DownloadModToCache(ctx context.Context, gameCache *cache.Cache } defer os.RemoveAll(stagePath) //nolint:errcheck - if game.DeployMode == domain.DeployCompile && isExmodzFile(file.FileName) { + if game.DeployMode == domain.DeployCompile && isExmodzFile(safeFileName) { compiler, ok := src.(source.Compiler) if !ok { return nil, fmt.Errorf("source %q: game %q requires DeployCompile but source does not implement Compiler", src.ID(), game.ID) @@ -548,7 +555,11 @@ func (s *Service) DownloadModToCache(ctx context.Context, gameCache *cache.Cache if err := os.MkdirAll(stagePath, 0755); err != nil { return nil, fmt.Errorf("preparing compile staging: %w", err) } - destName := compiledFileName(file.FileName) + // filepath.Base again: compiledFileName only trims a suffix/adds + // one - it does not strip directory components, so a traversal + // payload in safeFileName's stem would otherwise survive into + // destName unsanitized. + destName := filepath.Base(compiledFileName(safeFileName)) destPath := filepath.Join(stagePath, destName) if err := compiler.Compile(ctx, basePakPath, archivePath, destPath); err != nil { return nil, fmt.Errorf("compiling mod: %w", err) @@ -566,11 +577,11 @@ func (s *Service) DownloadModToCache(ctx context.Context, gameCache *cache.Cache // Copy mode: game wants files as-is (e.g., Hytale .zip mods) // Or not an archive - just copy to cache. copyFileStreaming mkdirs // stagePath itself (importer.go), so no MkdirAll needed here. - destPath := filepath.Join(stagePath, file.FileName) + destPath := filepath.Join(stagePath, safeFileName) if err := copyFileStreaming(archivePath, destPath); err != nil { return nil, fmt.Errorf("copying to cache: %w", err) } - if err := commitStagedCacheWithMarker(cachePath, stagePath, file.ID, []string{file.FileName}); err != nil { + if err := commitStagedCacheWithMarker(cachePath, stagePath, file.ID, []string{safeFileName}); err != nil { return nil, err } return &DownloadModResult{ @@ -666,11 +677,14 @@ func (s *Service) ingestLocalToCache(gameCache *cache.Cache, game *domain.Game, // item 12); localPath's own basename is often just a temp file name // and falls back only when the caller left FileName unset. // copyFileStreaming mkdirs stagePath itself (importer.go), so no - // MkdirAll needed here. + // MkdirAll needed here. filepath.Base sanitizes whichever name was + // chosen: file.FileName is SOURCE-CONTROLLED (#196 review) and must + // not be trusted as a path component verbatim (e.g. "../../evil"). destName := file.FileName if destName == "" { destName = filepath.Base(localPath) } + destName = filepath.Base(destName) if err := copyFileStreaming(localPath, filepath.Join(stagePath, destName)); err != nil { return nil, fmt.Errorf("copying to cache: %w", err) } diff --git a/internal/core/service_download_local_test.go b/internal/core/service_download_local_test.go index 99dfa57..4ec273a 100644 --- a/internal/core/service_download_local_test.go +++ b/internal/core/service_download_local_test.go @@ -238,6 +238,38 @@ func TestIngestLocalToCacheArchiveCopyModeUsesDeclaredFileName(t *testing.T) { assert.True(t, os.IsNotExist(err), "cached file must NOT be named after localPath's basename when file.FileName is declared") } +// TestIngestLocalToCacheArchiveCopyMode_TraversalFileNameSanitized is the +// ingestLocalToCache sibling of the #196 review traversal fix: file.FileName +// is SOURCE-CONTROLLED (any custom directory/manifest/api ModSource can +// declare it) and must never be trusted as a path component verbatim - an +// entry like "../evil.zip" must not let the cached file land outside the +// mod's own cache version directory. +func TestIngestLocalToCacheArchiveCopyMode_TraversalFileNameSanitized(t *testing.T) { + svc, gameCache := newLocalIngestService(t) + + tempFile := filepath.Join(t.TempDir(), "tmp-download-xyz.bin") + require.NoError(t, os.WriteFile(tempFile, []byte("zipbytes"), 0644)) + + game := &domain.Game{ID: "hytale", DeployMode: domain.DeployCopy} + mod := &domain.Mod{ID: "coolmod-2.0", SourceID: "my-mods", Version: "2.0"} + file := &domain.DownloadableFile{ID: "main", FileName: "../evil-traversal.zip"} + + result, err := svc.ingestLocalToCache(gameCache, game, mod, file, tempFile) + require.NoError(t, err) + assert.Equal(t, 1, result.FilesExtracted) + + sanitizedPath := gameCache.GetFilePath("hytale", "my-mods", "coolmod-2.0", "2.0", "evil-traversal.zip") + _, err = os.Stat(sanitizedPath) + assert.NoError(t, err, "the cached file must land under the sanitized (Base'd) name inside the version directory") + + // The version directory's PARENT (my-mods-coolmod-2.0/) is exactly one + // level up from the "2.0" version dir the unsanitized "../evil-*.zip" + // would have climbed into. + escapedPath := filepath.Join(gameCache.ModPath("hytale", "my-mods", "coolmod-2.0", "2.0"), "..", "evil-traversal.zip") + _, err = os.Stat(escapedPath) + assert.True(t, os.IsNotExist(err), "a traversal filename must never write outside the mod's own cache version directory") +} + // TestPrepareStagingCleansPartialStagingOnCopyFailure is a regression test // for a reviewer-caught behavior break in the #52 item 11 extraction: // pre-refactor, the caller armed `defer os.RemoveAll(stagePath)` BEFORE the diff --git a/internal/core/service_download_traversal_test.go b/internal/core/service_download_traversal_test.go new file mode 100644 index 0000000..f454d4f --- /dev/null +++ b/internal/core/service_download_traversal_test.go @@ -0,0 +1,113 @@ +package core_test + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/core" + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/stretchr/testify/require" +) + +// TestDownloadModToCache_TraversalFileName_SanitizedAgainstEscape is the +// download-path sibling of updater_test.go's ApplyRecompile traversal test +// (#196 review): DownloadModToCache is the MAIN download path, and its +// DownloadableFile.FileName is exactly as source-controlled as the +// redownload fallback's - a malicious or buggy source declaring a FileName +// like "../evil.zip" must never be able to write outside the intended +// staging/cache directories. +// +// DeployCopy exercises BOTH vulnerable joins in one call: the download's +// own archivePath (tempDir) and the copy-mode destPath (stagePath) that +// lands the file in the cache. +func TestDownloadModToCache_TraversalFileName_SanitizedAgainstEscape(t *testing.T) { + dataDir := t.TempDir() + cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: dataDir, CacheDir: t.TempDir()} + svc, err := core.NewService(cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + mock := newMockSourceWithDownloads("test") + defer mock.Close() + svc.RegisterSource(mock) + + game := &domain.Game{ID: "testgame", Name: "Test Game", ModPath: filepath.Join(t.TempDir(), "mods"), DeployMode: domain.DeployCopy} + require.NoError(t, svc.AddGame(game)) + + mod := &domain.Mod{ID: "123", SourceID: "test", Name: "Evil Mod", Version: "1.0.0", GameID: "testgame"} + file := &domain.DownloadableFile{ID: "file1", Name: "Evil File", FileName: "../evil-traversal.zip"} + mock.AddDownload(file.ID, []byte("payload")) + + result, err := svc.DownloadMod(context.Background(), "test", game, mod, file, nil) + require.NoError(t, err) + require.Equal(t, 1, result.FilesExtracted) + + // newStagingDir("lmm-download-*") creates its scratch dir directly under + // dataDir/downloads (Service.stagingRoot) - an UNSANITIZED + // filepath.Join(tempDir, "../evil-traversal.zip") climbs exactly one + // level out of that scratch dir, landing at dataDir/downloads/ + // evil-traversal.zip. That parent is never removed (only tempDir itself + // is), so an escaped write would persist right here. + escapedPath := filepath.Join(dataDir, "downloads", "evil-traversal.zip") + _, statErr := os.Stat(escapedPath) + require.True(t, os.IsNotExist(statErr), "a traversal filename must never write outside the staging tempDir") + + gameCache := svc.GetGameCache(game) + files, err := gameCache.ListFiles(game.ID, mod.SourceID, mod.ID, mod.Version) + require.NoError(t, err) + require.Equal(t, []string{"evil-traversal.zip"}, files, "the sanitized (Base'd) filename is what must actually land in the cache") +} + +// TestDownloadMod_DeployCompile_TraversalFileName_SanitizedAgainstEscape +// covers the third #196-review site in the same function: the DeployCompile +// branch derives destName via compiledFileName(file.FileName), which only +// trims/adds a suffix - it does not strip directory components, so a +// traversal payload in the STEM (e.g. "../evil.exmodz") survives into +// destName unless separately re-sanitized before the final +// filepath.Join(stagePath, destName). +func TestDownloadMod_DeployCompile_TraversalFileName_SanitizedAgainstEscape(t *testing.T) { + dlSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("fake-exmodz-bytes")) + })) + defer dlSrv.Close() + + installDir := t.TempDir() + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + writeFakeBasePak(t, basePak) + + cacheDir := t.TempDir() + cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: cacheDir} + svc, err := core.NewService(cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + src := &fakeCompilerSource{downloadURL: dlSrv.URL} + svc.RegisterSource(src) + + game := &domain.Game{ID: "icarus", InstallPath: installDir, ModPath: t.TempDir(), DeployMode: domain.DeployCompile} + require.NoError(t, svc.AddGame(game)) + + mod := &domain.Mod{ID: "bear-mount", SourceID: "fake-compiler", GameID: "icarus", Version: "3.3"} + file := &domain.DownloadableFile{ID: "exmodz", FileName: "../evil-traversal.exmodz"} + + result, err := svc.DownloadMod(context.Background(), "fake-compiler", game, mod, file, nil) + require.NoError(t, err) + require.Equal(t, 1, result.FilesExtracted) + + // The mod's own cache dir is cacheDir/icarus/fake-compiler-bear-mount/3.3 + // - an unsanitized "../evil-traversal" stem would climb into + // fake-compiler-bear-mount/ (one level up from the version dir). + gameCache := svc.GetGameCache(game) + escapedPath := filepath.Join(gameCache.ModPath(game.ID, mod.SourceID, mod.ID, mod.Version), "..", "evil-traversal_P.pak") + _, statErr := os.Stat(escapedPath) + require.True(t, os.IsNotExist(statErr), "a traversal filename's compiled output must never escape the version directory") + + files, err := gameCache.ListFiles(game.ID, mod.SourceID, mod.ID, mod.Version) + require.NoError(t, err) + require.Equal(t, []string{"evil-traversal_P.pak"}, files, "the sanitized (Base'd) compiled name is what must actually land in the cache") +} From c0d829bf95f1bd4f9008c7578349666237dc8722 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 20:37:58 -0400 Subject: [PATCH 65/96] feat: MergeCompiler interface + Icarus merge engine (#197) --- internal/source/icarus/icarus.go | 19 +- internal/source/icarus/merge.go | 150 ++++++++++++++ internal/source/icarus/merge_test.go | 288 +++++++++++++++++++++++++++ internal/source/source.go | 39 ++-- 4 files changed, 478 insertions(+), 18 deletions(-) create mode 100644 internal/source/icarus/merge.go create mode 100644 internal/source/icarus/merge_test.go diff --git a/internal/source/icarus/icarus.go b/internal/source/icarus/icarus.go index acdfb93..76848b0 100644 --- a/internal/source/icarus/icarus.go +++ b/internal/source/icarus/icarus.go @@ -34,14 +34,21 @@ func New(httpClient *http.Client, projectID string) *Icarus { var ( _ source.ModSource = (*Icarus)(nil) _ source.CapabilityReporter = (*Icarus)(nil) - _ source.Compiler = (*Icarus)(nil) + _ source.MergeCompiler = (*Icarus)(nil) ) -// Compile implements source.Compiler by delegating to the package-level -// Compile function. ctx is unused: compiling is pure local file I/O against -// the installed game's own pak (#175), with nothing to cancel. -func (s *Icarus) Compile(_ context.Context, basePakPath, sourceFilePath, outputPath string) error { - return Compile(basePakPath, sourceFilePath, outputPath) +// ValidateSource implements source.MergeCompiler by delegating to the +// package-level ValidateSource function. +func (s *Icarus) ValidateSource(sourceFilePath string) error { + return ValidateSource(sourceFilePath) +} + +// MergeCompile implements source.MergeCompiler by delegating to the +// package-level MergeCompile function. ctx is unused: merging is pure local +// file I/O against the installed game's own pak (#175/#197), with nothing +// to cancel. +func (s *Icarus) MergeCompile(ctx context.Context, basePakPath string, sources []MergeSource, outputPakPath string) ([]string, error) { + return MergeCompile(ctx, basePakPath, sources, outputPakPath) } func (s *Icarus) ID() string { return "icarus" } diff --git a/internal/source/icarus/merge.go b/internal/source/icarus/merge.go new file mode 100644 index 0000000..888880e --- /dev/null +++ b/internal/source/icarus/merge.go @@ -0,0 +1,150 @@ +package icarus + +import ( + "context" + "fmt" + "os" + + "github.com/DonovanMods/linux-mod-manager/internal/source" + "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" +) + +// MergeSource is a type alias (not a distinct type) for source.MergeSource +// (Step 3 above). internal/core must NOT import this icarus package +// directly (established #136/#196 precedent - see +// service_icarus_compile_test.go's fakeCompilerSource doc comment), so it +// can only ever construct/consume source.MergeSource values - aliasing it +// here, rather than defining a second, structurally-similar type, is what +// lets *Icarus's MergeCompile method (Step 6) satisfy source.MergeCompiler +// at all: Go interface satisfaction requires identical types, and a type +// alias IS the same type, not a look-alike. +type MergeSource = source.MergeSource + +// ValidateSource parses exmodzPath without compiling anything - the +// ingest-time check (#197 design: "install still parses/validates the +// .exmodz early"). A malformed archive fails loud immediately, at +// download/import time, rather than at the next merge (which may not run +// until a later mutation). +func ValidateSource(exmodzPath string) error { + data, err := os.ReadFile(exmodzPath) + if err != nil { + return fmt.Errorf("icarus: reading %s: %w", exmodzPath, err) + } + if _, err := ParseExmodz(data); err != nil { + return fmt.Errorf("icarus: validating %s: %w", exmodzPath, err) + } + return nil +} + +// MergeCompile applies every source's .EXMOD row upserts, IN ORDER, against +// the same evolving base tables - a merge is just Compile with N diffs +// instead of 1. Table conflicts compose at the FIELD level for free: +// ApplyRowPatch always shallow-merges an item's fields into whatever the +// target row currently holds, so feeding mod A's patched bytes back in as +// the "base" for mod B's row (instead of re-reading the pristine base table +// each time) is the entire merge algorithm - two mods patching DIFFERENT +// fields of the same row, or entirely different rows of the same table, +// both survive; only a genuine same-row-same-field write is last-wins (an +// ordinary, expected upsert outcome, not something to warn about). Bundled +// ASSET files cannot compose this way - a same-path asset collision is +// necessarily last-wins, so it is reported as a warning instead. +// +// ctx is accepted only to satisfy source.MergeCompiler and is never read - +// every step here is local file I/O over small files (mirrors Compile's own +// doc comment, internal/source/icarus/compile.go:23-25). +// +// A non-nil error always means outputPakPath does not exist (or does not +// contain a fully-written pak) - see the removal defer below, mirroring +// Compile's own fail-clean contract. +func MergeCompile(ctx context.Context, basePakPath string, sources []MergeSource, outputPakPath string) (warnings []string, err error) { + base, err := unrealpak.Open(basePakPath) + if err != nil { + return nil, fmt.Errorf("icarus: opening base pak %s: %w", basePakPath, err) + } + defer base.Close() //nolint:errcheck + + tableState := make(map[string][]byte) // mountPath -> current (possibly already patched) JSON bytes + assets := make(map[string][]byte) // final asset path -> data (last source wins) + assetOwner := make(map[string]string) // asset path -> ModRef that last set it + + for _, src := range sources { + exmodzData, rerr := os.ReadFile(src.ExmodzPath) + if rerr != nil { + return warnings, fmt.Errorf("icarus: reading %s: %w", src.ExmodzPath, rerr) + } + bundle, perr := ParseExmodz(exmodzData) + if perr != nil { + return warnings, fmt.Errorf("icarus: %s: %w", src.ExmodzPath, perr) + } + + for _, row := range bundle.Diff.Rows { + if row.CurrentFile == endOfModSentinel { + continue + } + if len(row.FileItems) == 0 { + return warnings, fmt.Errorf("icarus: %s: row has no File_Items to apply (malformed .EXMOD manifest)", row.CurrentFile) + } + mountPath, merr := resolveCurrentFile(base, row.CurrentFile) + if merr != nil { + return warnings, merr + } + current, seen := tableState[mountPath] + if !seen { + current, merr = base.ReadFile(mountPath) + if merr != nil { + return warnings, fmt.Errorf("icarus: reading base data table %s: %w", mountPath, merr) + } + } + patched, perr2 := ApplyRowPatch(current, row) + if perr2 != nil { + return warnings, perr2 + } + tableState[mountPath] = patched + } + + for assetPath, data := range bundle.Assets { + safePath, serr := sanitizeAssetPath(assetPath) + if serr != nil { + return warnings, serr + } + if owner, exists := assetOwner[safePath]; exists && owner != src.ModRef { + warnings = append(warnings, fmt.Sprintf( + "asset %q is bundled by both %s and %s - %s wins (last-applied, per profile load order)", + safePath, owner, src.ModRef, src.ModRef)) + } + assets[safePath] = data + assetOwner[safePath] = src.ModRef + } + } + + out, cerr := unrealpak.Create(outputPakPath, unrealpak.WithMountPoint(icarusContentMountPoint)) + if cerr != nil { + return warnings, fmt.Errorf("icarus: creating %s: %w", outputPakPath, cerr) + } + defer func() { + if err == nil { + return + } + _ = out.Close() //nolint:errcheck + if rmErr := os.Remove(outputPakPath); rmErr != nil && !os.IsNotExist(rmErr) { + err = fmt.Errorf("%w (additionally, removing partial output %s failed: %v)", err, outputPakPath, rmErr) + } + }() + + for mountPath, data := range tableState { + tablePath := icarusDataTablePrefix + mountPath + if err = out.AddFile(tablePath, data); err != nil { + return warnings, fmt.Errorf("icarus: writing merged %s: %w", tablePath, err) + } + } + for assetPath, data := range assets { + if err = out.AddFile(assetPath, data); err != nil { + return warnings, fmt.Errorf("icarus: writing bundled asset %s: %w", assetPath, err) + } + } + + if err = out.Close(); err != nil { + return warnings, fmt.Errorf("icarus: finalizing %s: %w", outputPakPath, err) + } + return warnings, nil +} diff --git a/internal/source/icarus/merge_test.go b/internal/source/icarus/merge_test.go new file mode 100644 index 0000000..8917011 --- /dev/null +++ b/internal/source/icarus/merge_test.go @@ -0,0 +1,288 @@ +package icarus + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/source" + "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" +) + +// TestMergeCompile_FieldLevelMergeAcrossMods is the crux of #197: two mods +// patch DIFFERENT fields of the SAME row in the SAME table. Whole-pak +// last-wins (the #136 status quo) would lose one mod's field entirely; +// sequential upserts must preserve BOTH. +func TestMergeCompile_FieldLevelMergeAcrossMods(t *testing.T) { + baseTables := map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Rows":[{"Name":"Mount_Bear","BaseMovementSpeed":200,"BaseHealth":500}]}`), + } + basePak := writeTestBasePak(t, baseTables) + + modA := writeTestExmodzFile(t, `{"name":"Speed Mod","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":235}]}]}`, nil) + modB := writeTestExmodzFile(t, `{"name":"Health Mod","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseHealth":800}]}]}`, nil) + + outputPath := filepath.Join(t.TempDir(), "merged_P.pak") + warnings, err := MergeCompile(context.Background(), basePak, []source.MergeSource{ + {ModRef: "icarus:speed-mod", ExmodzPath: modA}, + {ModRef: "icarus:health-mod", ExmodzPath: modB}, + }, outputPath) + if err != nil { + t.Fatalf("MergeCompile: %v", err) + } + if len(warnings) != 0 { + t.Errorf("warnings = %v, want none (no asset collision in this fixture)", warnings) + } + + r, err := unrealpak.Open(outputPath) + if err != nil { + t.Fatalf("opening merged output: %v", err) + } + defer r.Close() //nolint:errcheck + + merged, err := r.ReadFile("data/AI/D_AIGrowth.json") + if err != nil { + t.Fatalf("ReadFile merged data table: %v", err) + } + if !bytes.Contains(merged, []byte(`"BaseMovementSpeed":235`)) { + t.Errorf("merged table = %s, want BaseMovementSpeed 235 (mod A's field) to survive", merged) + } + if !bytes.Contains(merged, []byte(`"BaseHealth":800`)) { + t.Errorf("merged table = %s, want BaseHealth 800 (mod B's field) to survive", merged) + } +} + +// TestMergeCompile_DifferentTablesFromDifferentMods proves the OTHER +// whole-pak-last-wins failure mode (#197's issue body point 1): mod A +// patches table X, mod B patches table Y - both must land in the single +// merged pak, not just the last mod's table. +func TestMergeCompile_DifferentTablesFromDifferentMods(t *testing.T) { + baseTables := map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Rows":[{"Name":"Mount_Bear","BaseMovementSpeed":200}]}`), + "Items/D_ItemsStatic.json": []byte(`{"Rows":[{"Name":"Item_Saddle","Weight":5}]}`), + } + basePak := writeTestBasePak(t, baseTables) + + modA := writeTestExmodzFile(t, `{"name":"Mount Mod","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":300}]}]}`, nil) + modB := writeTestExmodzFile(t, `{"name":"Item Mod","Rows":[{"CurrentFile":"Items-D_ItemsStatic.json","File_Items":[{"Name":"Item_Saddle","Weight":1}]}]}`, nil) + + outputPath := filepath.Join(t.TempDir(), "merged_P.pak") + if _, err := MergeCompile(context.Background(), basePak, []source.MergeSource{ + {ModRef: "icarus:mount-mod", ExmodzPath: modA}, + {ModRef: "icarus:item-mod", ExmodzPath: modB}, + }, outputPath); err != nil { + t.Fatalf("MergeCompile: %v", err) + } + + r, err := unrealpak.Open(outputPath) + if err != nil { + t.Fatalf("opening merged output: %v", err) + } + defer r.Close() //nolint:errcheck + + aiTable, err := r.ReadFile("data/AI/D_AIGrowth.json") + if err != nil { + t.Fatalf("ReadFile AI table: %v", err) + } + if !bytes.Contains(aiTable, []byte(`"BaseMovementSpeed":300`)) { + t.Errorf("AI table = %s, want mod A's patch", aiTable) + } + itemsTable, err := r.ReadFile("data/Items/D_ItemsStatic.json") + if err != nil { + t.Fatalf("ReadFile Items table: %v", err) + } + if !bytes.Contains(itemsTable, []byte(`"Weight":1`)) { + t.Errorf("Items table = %s, want mod B's patch", itemsTable) + } +} + +// TestMergeCompile_SameRowSameField_LastWins pins the EXPECTED (not +// warned-about) outcome when two mods genuinely conflict on the exact same +// field of the exact same row: later-in-order wins, ordinary upsert +// semantics, no special handling needed. +func TestMergeCompile_SameRowSameField_LastWins(t *testing.T) { + baseTables := map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Rows":[{"Name":"Mount_Bear","BaseMovementSpeed":200}]}`), + } + basePak := writeTestBasePak(t, baseTables) + + modA := writeTestExmodzFile(t, `{"name":"A","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":300}]}]}`, nil) + modB := writeTestExmodzFile(t, `{"name":"B","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":400}]}]}`, nil) + + outputPath := filepath.Join(t.TempDir(), "merged_P.pak") + if _, err := MergeCompile(context.Background(), basePak, []source.MergeSource{ + {ModRef: "icarus:a", ExmodzPath: modA}, + {ModRef: "icarus:b", ExmodzPath: modB}, + }, outputPath); err != nil { + t.Fatalf("MergeCompile: %v", err) + } + + r, err := unrealpak.Open(outputPath) + if err != nil { + t.Fatalf("opening merged output: %v", err) + } + defer r.Close() //nolint:errcheck + merged, err := r.ReadFile("data/AI/D_AIGrowth.json") + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if !bytes.Contains(merged, []byte(`"BaseMovementSpeed":400`)) { + t.Errorf("merged table = %s, want mod B's (later, order-2) value 400 to win", merged) + } + if bytes.Contains(merged, []byte(`"BaseMovementSpeed":300`)) { + t.Errorf("merged table = %s, mod A's value should have been overwritten", merged) + } +} + +// TestMergeCompile_AssetCollision_LastWinsWithWarning: two mods bundle a +// prebuilt asset at the SAME path - cannot compose like a table row, so +// last-applied wins AND a warning is returned. +func TestMergeCompile_AssetCollision_LastWinsWithWarning(t *testing.T) { + basePak := writeTestBasePak(t, map[string][]byte{"AI/D_AIGrowth.json": []byte(`{"Rows":[]}`)}) + + modA := writeTestExmodzFile(t, `{"name":"A","Rows":[]}`, map[string][]byte{ + "Shared/ASS/SK_Shared.uasset": []byte("from-mod-a"), + }) + modB := writeTestExmodzFile(t, `{"name":"B","Rows":[]}`, map[string][]byte{ + "Shared/ASS/SK_Shared.uasset": []byte("from-mod-b"), + }) + + outputPath := filepath.Join(t.TempDir(), "merged_P.pak") + warnings, err := MergeCompile(context.Background(), basePak, []source.MergeSource{ + {ModRef: "icarus:a", ExmodzPath: modA}, + {ModRef: "icarus:b", ExmodzPath: modB}, + }, outputPath) + if err != nil { + t.Fatalf("MergeCompile: %v", err) + } + if len(warnings) != 1 { + t.Fatalf("warnings = %v, want exactly 1 asset-collision warning", warnings) + } + if !bytes.Contains([]byte(warnings[0]), []byte("Shared/ASS/SK_Shared.uasset")) { + t.Errorf("warning = %q, want it to name the colliding path", warnings[0]) + } + if !bytes.Contains([]byte(warnings[0]), []byte("icarus:b")) { + t.Errorf("warning = %q, want it to name the winning mod", warnings[0]) + } + + r, err := unrealpak.Open(outputPath) + if err != nil { + t.Fatalf("opening merged output: %v", err) + } + defer r.Close() //nolint:errcheck + asset, err := r.ReadFile("Shared/ASS/SK_Shared.uasset") + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(asset) != "from-mod-b" { + t.Errorf("asset content = %q, want mod B's (later-applied) content to win", asset) + } +} + +// TestMergeCompile_ContentAddingModComposesWithPatchMod: one mod ADDS a +// brand-new row (a new mountable species), another PATCHES an existing row +// in the SAME table. Both must survive in the merged output. +func TestMergeCompile_ContentAddingModComposesWithPatchMod(t *testing.T) { + baseTables := map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Rows":[{"Name":"Mount_Bear","BaseMovementSpeed":200}]}`), + } + basePak := writeTestBasePak(t, baseTables) + + patchMod := writeTestExmodzFile(t, `{"name":"Patch","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":250}]}]}`, nil) + addMod := writeTestExmodzFile(t, `{"name":"NewSpecies","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Wolf","BaseMovementSpeed":320}]}]}`, nil) + + outputPath := filepath.Join(t.TempDir(), "merged_P.pak") + if _, err := MergeCompile(context.Background(), basePak, []source.MergeSource{ + {ModRef: "icarus:patch", ExmodzPath: patchMod}, + {ModRef: "icarus:add", ExmodzPath: addMod}, + }, outputPath); err != nil { + t.Fatalf("MergeCompile: %v", err) + } + + r, err := unrealpak.Open(outputPath) + if err != nil { + t.Fatalf("opening merged output: %v", err) + } + defer r.Close() //nolint:errcheck + merged, err := r.ReadFile("data/AI/D_AIGrowth.json") + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if !bytes.Contains(merged, []byte(`"BaseMovementSpeed":250`)) { + t.Errorf("merged table = %s, want the patched Mount_Bear speed", merged) + } + if !bytes.Contains(merged, []byte(`"Mount_Wolf"`)) { + t.Errorf("merged table = %s, want the newly-added Mount_Wolf row", merged) + } +} + +// TestMergeCompile_SingleSource_MatchesCompile proves the N=1 degenerate +// case (a profile with exactly one enabled exmodz mod) produces byte- +// identical table content to the existing single-mod Compile() - the +// merged-only model must not regress the already-shipped single-mod path. +func TestMergeCompile_SingleSource_MatchesCompile(t *testing.T) { + baseTables := map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Rows":[{"Name":"Mount_Bear","BaseMovementSpeed":200}]}`), + } + basePak := writeTestBasePak(t, baseTables) + manifest := `{"name":"Bear Mount","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":235}]}]}` + exmodzPath := writeTestExmodzFile(t, manifest, map[string][]byte{ + "Bear_Mount/ASS/ITM/SK_ITM_Saddle_Bear.uasset": []byte("fake-asset"), + }) + + compileOut := filepath.Join(t.TempDir(), "compile_P.pak") + if err := Compile(basePak, exmodzPath, compileOut); err != nil { + t.Fatalf("Compile: %v", err) + } + mergeOut := filepath.Join(t.TempDir(), "merge_P.pak") + if _, err := MergeCompile(context.Background(), basePak, []source.MergeSource{{ModRef: "icarus:bear-mount", ExmodzPath: exmodzPath}}, mergeOut); err != nil { + t.Fatalf("MergeCompile: %v", err) + } + + cr, err := unrealpak.Open(compileOut) + if err != nil { + t.Fatalf("opening Compile output: %v", err) + } + defer cr.Close() //nolint:errcheck + mr, err := unrealpak.Open(mergeOut) + if err != nil { + t.Fatalf("opening MergeCompile output: %v", err) + } + defer mr.Close() //nolint:errcheck + + cTable, err := cr.ReadFile("data/AI/D_AIGrowth.json") + if err != nil { + t.Fatalf("Compile ReadFile: %v", err) + } + mTable, err := mr.ReadFile("data/AI/D_AIGrowth.json") + if err != nil { + t.Fatalf("MergeCompile ReadFile: %v", err) + } + if !bytes.Equal(cTable, mTable) { + t.Errorf("Compile table = %s, MergeCompile table = %s, want identical for N=1", cTable, mTable) + } +} + +// TestValidateSource_ValidExmodz_NoError proves ValidateSource accepts a +// well-formed .exmodz without compiling anything (no basePak needed). +func TestValidateSource_ValidExmodz_NoError(t *testing.T) { + exmodzPath := writeTestExmodzFile(t, `{"name":"OK","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":200}]}]}`, nil) + if err := ValidateSource(exmodzPath); err != nil { + t.Errorf("ValidateSource: %v, want nil for a well-formed .exmodz", err) + } +} + +// TestValidateSource_MalformedExmodz_Errors proves a corrupt/unparseable +// .exmodz fails loud at validate time (ingest-time), not silently deferred +// to the next merge. +func TestValidateSource_MalformedExmodz_Errors(t *testing.T) { + path := filepath.Join(t.TempDir(), "bad.exmodz") + if err := os.WriteFile(path, []byte("not a zip file"), 0o644); err != nil { + t.Fatal(err) + } + if err := ValidateSource(path); err == nil { + t.Error("ValidateSource: got nil error, want a failure for a non-zip file") + } +} diff --git a/internal/source/source.go b/internal/source/source.go index ad95693..81e9f06 100644 --- a/internal/source/source.go +++ b/internal/source/source.go @@ -143,16 +143,31 @@ type DownloadHeaderProvider interface { DownloadHeaders(fileURL string) map[string]string } -// Compiler is implemented by sources whose downloaded files need -// transforming into a different artifact before deployment (Icarus's -// .exmodz -> .pak). Service consults it, when DeployMode is DeployCompile, -// after downloading but before committing the file to cache — the result -// replaces the downloaded file in cache, so everything downstream (Install, -// the linker) treats it exactly like a DeployCopy file. -// -// basePakPath is resolved by the caller from game.InstallPath; sourceFilePath -// is the just-downloaded file; outputPath is where the compiled result must be -// written. -type Compiler interface { - Compile(ctx context.Context, basePakPath, sourceFilePath, outputPath string) error +// MergeCompiler is implemented by sources whose compile-eligible files must +// be merged across every enabled mod into ONE profile-level artifact rather +// than compiled per-mod (#197: Icarus's cross-mod table merge - a whole-pak +// last-wins deploy would silently drop one mod's table rows whenever two +// mods patch the same table). Replaces #196's Compiler interface, which +// this source no longer implements: there is no more per-mod compiled +// artifact to produce. +type MergeCompiler interface { + // ValidateSource parses/validates sourceFilePath (the retained, + // not-yet-merged source archive) without compiling anything - called at + // ingest time (download/import) so a malformed archive fails loud + // immediately rather than at the next merge. + ValidateSource(sourceFilePath string) error + + // MergeCompile applies every entry in sources, in order (profile load + // order), against basePakPath's tables, and writes the merged result to + // outputPakPath. Returns non-fatal warnings (e.g. same-path asset + // collisions - last-applied wins) alongside a nil error; a nil error + // with warnings is still a fully-written, deployable pak. + MergeCompile(ctx context.Context, basePakPath string, sources []MergeSource, outputPakPath string) (warnings []string, err error) +} + +// MergeSource identifies one mod's contribution to a merge, in the order it +// must be applied (profile load order). +type MergeSource struct { + ModRef string // "sourceID:modID" - identity used in collision warnings + ExmodzPath string // the retained source archive to read } From c2b58104229e6704cbf78e3f3fbee537b1d1e000 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 20:41:32 -0400 Subject: [PATCH 66/96] feat: download path ingests .exmodz as validate+retain, no per-mod pak (#197) --- internal/core/service.go | 68 +++--- .../core/service_compile_fingerprint_test.go | 129 ----------- internal/core/service_icarus_compile_test.go | 212 ++++++------------ 3 files changed, 100 insertions(+), 309 deletions(-) delete mode 100644 internal/core/service_compile_fingerprint_test.go diff --git a/internal/core/service.go b/internal/core/service.go index 111556e..d43f6a9 100644 --- a/internal/core/service.go +++ b/internal/core/service.go @@ -167,34 +167,35 @@ func (s *Service) SourcesForGame(gameID string) ([]source.ModSource, error) { } // compilerSourceForGame resolves the sole Compiler-capable source -// registered for gameID (#173). The download path pins its Compiler check -// to the specific source a file was downloaded from (DownloadModToCache's -// src.(source.Compiler) check); Importer.Import has no such per-archive -// source to key off of, so it resolves against every source the game maps -// in its registry instead — matching resolveBasePak's v1 scope of "Icarus -// only", at most one of a game's configured sources implements Compiler -// today. Zero is the expected failure when the game (or its Compiler -// source) isn't configured; more than one is treated as ambiguous rather -// than picking arbitrarily — both fail loud instead of letting an .exmodz -// import silently skip compilation. -func (s *Service) compilerSourceForGame(gameID string) (source.Compiler, error) { +// registered for gameID (#173). The download path pins its MergeCompiler +// check to the specific source a file was downloaded from +// (DownloadModToCache's src.(source.MergeCompiler) check); Importer.Import +// has no such per-archive source to key off of, so it resolves against +// every source the game maps in its registry instead — matching +// resolveBasePak's v1 scope of "Icarus only", at most one of a game's +// configured sources implements MergeCompiler today. Zero is the expected +// failure when the game (or its MergeCompiler source) isn't configured; +// more than one is treated as ambiguous rather than picking arbitrarily — +// both fail loud instead of letting an .exmodz import silently skip +// validation. +func (s *Service) mergeCompilerSourceForGame(gameID string) (source.MergeCompiler, error) { srcs, err := s.SourcesForGame(gameID) if err != nil { return nil, err } - var compilers []source.Compiler + var compilers []source.MergeCompiler for _, src := range srcs { - if c, ok := src.(source.Compiler); ok { + if c, ok := src.(source.MergeCompiler); ok { compilers = append(compilers, c) } } switch len(compilers) { case 0: - return nil, fmt.Errorf("game %q requires DeployCompile but has no compiler-capable source configured (map a source implementing source.Compiler in the game's sources)", gameID) + return nil, fmt.Errorf("game %q requires DeployCompile but has no merge-compiler-capable source configured (map a source implementing source.MergeCompiler in the game's sources)", gameID) case 1: return compilers[0], nil default: - return nil, fmt.Errorf("game %q has multiple compiler-capable sources configured; ambiguous compile source", gameID) + return nil, fmt.Errorf("game %q has multiple merge-compiler-capable sources configured; ambiguous compile source", gameID) } } @@ -541,36 +542,31 @@ func (s *Service) DownloadModToCache(ctx context.Context, gameCache *cache.Cache defer os.RemoveAll(stagePath) //nolint:errcheck if game.DeployMode == domain.DeployCompile && isExmodzFile(safeFileName) { - compiler, ok := src.(source.Compiler) + mc, ok := src.(source.MergeCompiler) if !ok { - return nil, fmt.Errorf("source %q: game %q requires DeployCompile but source does not implement Compiler", src.ID(), game.ID) + return nil, fmt.Errorf("source %q: game %q requires DeployCompile but source does not implement MergeCompiler", src.ID(), game.ID) } - basePakPath, err := resolveBasePak(game) - if err != nil { - return nil, err + if err := mc.ValidateSource(archivePath); err != nil { + return nil, fmt.Errorf("validating %s: %w", safeFileName, err) } // Unlike copyFileStreaming (which mkdirs its destination itself), - // Compile writes via unrealpak.Create - a bare os.Create - so - // stagePath must exist before it's called. + // the retained-source write below needs stagePath to exist first. if err := os.MkdirAll(stagePath, 0755); err != nil { - return nil, fmt.Errorf("preparing compile staging: %w", err) + return nil, fmt.Errorf("preparing staging: %w", err) } - // filepath.Base again: compiledFileName only trims a suffix/adds - // one - it does not strip directory components, so a traversal - // payload in safeFileName's stem would otherwise survive into - // destName unsanitized. - destName := filepath.Base(compiledFileName(safeFileName)) - destPath := filepath.Join(stagePath, destName) - if err := compiler.Compile(ctx, basePakPath, archivePath, destPath); err != nil { - return nil, fmt.Errorf("compiling mod: %w", err) - } - if err := stageCompileFingerprint(stagePath, file.ID, basePakPath, archivePath); err != nil { - return nil, err + retainedPath := filepath.Join(stagePath, cache.RetainedSourceName(file.ID)) + if err := copyFileStreaming(archivePath, retainedPath); err != nil { + return nil, fmt.Errorf("retaining %s: %w", safeFileName, err) } - if err := commitStagedCacheWithMarker(cachePath, stagePath, file.ID, []string{destName}); err != nil { + // members is nil (#197): this cache entry's ONLY content is the + // reserved retained source - there is no per-mod deployment + // artifact anymore. The merged pak (a separate, profile-level + // cache entry - internal/core/merged_pak.go) is what actually + // deploys. + if err := commitStagedCacheWithMarker(cachePath, stagePath, file.ID, nil); err != nil { return nil, err } - return &DownloadModResult{FilesExtracted: 1, Checksum: downloadResult.Checksum}, nil + return &DownloadModResult{FilesExtracted: 0, Checksum: downloadResult.Checksum}, nil } if game.DeployMode == domain.DeployCopy || !s.extractor.CanExtract(archivePath) { diff --git a/internal/core/service_compile_fingerprint_test.go b/internal/core/service_compile_fingerprint_test.go deleted file mode 100644 index c5778b4..0000000 --- a/internal/core/service_compile_fingerprint_test.go +++ /dev/null @@ -1,129 +0,0 @@ -package core_test - -import ( - "context" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "testing" - - "github.com/DonovanMods/linux-mod-manager/internal/core" - "github.com/DonovanMods/linux-mod-manager/internal/domain" - "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" - "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" - "github.com/stretchr/testify/require" -) - -// basePakIndexHash opens path and returns its footer IndexHash - the same -// value Service's compile branches record, computed independently here so -// tests can assert against it without depending on internal/core internals. -func basePakIndexHash(t *testing.T, path string) string { - t.Helper() - r, err := unrealpak.Open(path) - require.NoError(t, err) - defer r.Close() //nolint:errcheck - return r.IndexHash() -} - -// TestDownloadMod_DeployCompile_RecordsBaseIndexHashAndRetainedSource pins -// #196 design points 1-2 for the DOWNLOAD compile path: compiling an -// .exmodz must record the base pak's IndexHash under the file's real -// DownloadableFile.ID, retain the original .exmodz bytes beside the -// compiled pak, and keep both out of ListFiles/deploy. -func TestDownloadMod_DeployCompile_RecordsBaseIndexHashAndRetainedSource(t *testing.T) { - dlSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write([]byte("original-exmodz-bytes")) - })) - defer dlSrv.Close() - - installDir := t.TempDir() - basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") - require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) - writeFakeBasePak(t, basePak) - wantHash := basePakIndexHash(t, basePak) - - cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} - svc, err := core.NewService(cfg) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, svc.Close()) }) - - src := &fakeCompilerSource{downloadURL: dlSrv.URL} - svc.RegisterSource(src) - - game := &domain.Game{ID: "icarus", InstallPath: installDir, ModPath: t.TempDir(), DeployMode: domain.DeployCompile} - require.NoError(t, svc.AddGame(game)) - - mod := &domain.Mod{ID: "bear-mount", SourceID: "fake-compiler", GameID: "icarus", Version: "3.3"} - file := &domain.DownloadableFile{ID: "exmodz-file-id", FileName: "Bear_Mount.exmodz"} - - _, err = svc.DownloadMod(context.Background(), "fake-compiler", game, mod, file, nil) - require.NoError(t, err) - - gameCache := svc.GetGameCache(game) - - hashes, err := gameCache.BaseIndexHashes(game.ID, mod.SourceID, mod.ID, mod.Version) - require.NoError(t, err) - require.Equal(t, map[string]string{"exmodz-file-id": wantHash}, hashes) - - retainedPath := gameCache.GetFilePath(game.ID, mod.SourceID, mod.ID, mod.Version, cache.RetainedSourceName("exmodz-file-id")) - retainedData, err := os.ReadFile(retainedPath) - require.NoError(t, err) - require.Equal(t, "original-exmodz-bytes", string(retainedData)) - - files, err := gameCache.ListFiles(game.ID, mod.SourceID, mod.ID, mod.Version) - require.NoError(t, err) - require.Equal(t, []string{"Bear_Mount_P.pak"}, files, "retained source and base-index marker must never be deployable content") -} - -// TestImportMod_DeployCompile_RecordsBaseIndexHashAndRetainedSource mirrors -// the above for the IMPORT compile path (keyed by the compiled output's own -// filename, since Import has no real DownloadableFile.ID available at -// compile time - see stageCompileFingerprint's doc comment). -func TestImportMod_DeployCompile_RecordsBaseIndexHashAndRetainedSource(t *testing.T) { - installDir := t.TempDir() - basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") - require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) - writeFakeBasePak(t, basePak) - wantHash := basePakIndexHash(t, basePak) - - cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} - svc, err := core.NewService(cfg) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, svc.Close()) }) - - src := &fakeCompilerSource{} - svc.RegisterSource(src) - - game := &domain.Game{ - ID: "icarus", - InstallPath: installDir, - ModPath: t.TempDir(), - DeployMode: domain.DeployCompile, - SourceIDs: map[string]string{"fake-compiler": "external-icarus-id"}, - } - require.NoError(t, svc.AddGame(game)) - - tempDir := t.TempDir() - archivePath := filepath.Join(tempDir, "Bear_Mount.exmodz") - require.NoError(t, os.WriteFile(archivePath, []byte("original-exmodz-bytes"), 0o644)) - - importer := svc.NewImporter(game) - result, err := importer.Import(context.Background(), archivePath, game, core.ImportOptions{}) - require.NoError(t, err) - - gameCache := svc.GetGameCache(game) - - hashes, err := gameCache.BaseIndexHashes(game.ID, result.Mod.SourceID, result.Mod.ID, result.Mod.Version) - require.NoError(t, err) - require.Equal(t, map[string]string{"Bear_Mount_P.pak": wantHash}, hashes) - - retainedPath := gameCache.GetFilePath(game.ID, result.Mod.SourceID, result.Mod.ID, result.Mod.Version, cache.RetainedSourceName("Bear_Mount_P.pak")) - retainedData, err := os.ReadFile(retainedPath) - require.NoError(t, err) - require.Equal(t, "original-exmodz-bytes", string(retainedData)) - - files, err := gameCache.ListFiles(game.ID, result.Mod.SourceID, result.Mod.ID, result.Mod.Version) - require.NoError(t, err) - require.Equal(t, []string{"Bear_Mount_P.pak"}, files, "retained source and base-index marker must never be deployable content") -} diff --git a/internal/core/service_icarus_compile_test.go b/internal/core/service_icarus_compile_test.go index 5dd29aa..7856254 100644 --- a/internal/core/service_icarus_compile_test.go +++ b/internal/core/service_icarus_compile_test.go @@ -2,6 +2,7 @@ package core_test import ( "context" + "fmt" "net/http" "net/http/httptest" "os" @@ -11,6 +12,7 @@ import ( "github.com/DonovanMods/linux-mod-manager/internal/core" "github.com/DonovanMods/linux-mod-manager/internal/domain" "github.com/DonovanMods/linux-mod-manager/internal/source" + "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" "github.com/stretchr/testify/require" ) @@ -29,13 +31,14 @@ func writeFakeBasePak(t *testing.T, path string) { } // fakeCompilerSource is a minimal ModSource that also implements -// source.Compiler, standing in for internal/source/icarus.Icarus (Tasks -// 8/13) without pulling that package into internal/core's tests — this test -// only needs to prove Service invokes Compile when DeployMode is -// DeployCompile, which Task 12 already tests in isolation. +// source.MergeCompiler, standing in for internal/source/icarus.Icarus +// without pulling that package into internal/core's tests — this test only +// needs to prove Service validates and retains (never compiles a per-mod +// pak) when DeployMode is DeployCompile (#197: merged-only). type fakeCompilerSource struct { - downloadURL string - compileCalls int + downloadURL string + compileCalls int + validateCalls int } func (s *fakeCompilerSource) ID() string { return "fake-compiler" } @@ -63,25 +66,54 @@ func (s *fakeCompilerSource) CheckUpdates(ctx context.Context, installed []domai return nil, source.ErrNotSupported } -// Compile implements source.Compiler by copying the downloaded source file -// through unchanged — this test only asserts Service invoked it with the -// right arguments and used its output, not that it performs real PAK -// compilation (Task 12 covers that). -func (s *fakeCompilerSource) Compile(ctx context.Context, basePakPath, sourceFilePath, outputPath string) error { - s.compileCalls++ - data, err := os.ReadFile(sourceFilePath) - if err != nil { +// ValidateSource implements source.MergeCompiler by confirming the archive +// exists — this test only asserts Service invoked it, not that it performs +// real .exmodz parsing (Task 1 covers that in the icarus package itself). +func (s *fakeCompilerSource) ValidateSource(sourceFilePath string) error { + s.validateCalls++ + if _, err := os.Stat(sourceFilePath); err != nil { return err } - return os.WriteFile(outputPath, data, 0o644) + return nil +} + +// MergeCompile implements source.MergeCompiler by concatenating every +// source's bytes - enough for tests to distinguish "which sources were +// actually merged" without needing a real base pak table to patch. +func (s *fakeCompilerSource) MergeCompile(ctx context.Context, basePakPath string, sources []source.MergeSource, outputPath string) ([]string, error) { + s.compileCalls++ + var out []byte + for _, src := range sources { + data, err := os.ReadFile(src.ExmodzPath) + if err != nil { + return nil, err + } + out = append(out, data...) + } + return nil, os.WriteFile(outputPath, out, 0o644) } var ( - _ source.ModSource = (*fakeCompilerSource)(nil) - _ source.Compiler = (*fakeCompilerSource)(nil) + _ source.ModSource = (*fakeCompilerSource)(nil) + _ source.MergeCompiler = (*fakeCompilerSource)(nil) ) -func TestDownloadMod_DeployCompile_InvokesCompiler(t *testing.T) { +// failingValidateCompilerSource wraps fakeCompilerSource and always fails +// ValidateSource - simulates a corrupt/malformed downloaded .exmodz. +type failingValidateCompilerSource struct { + *fakeCompilerSource +} + +func (s *failingValidateCompilerSource) ValidateSource(sourceFilePath string) error { + return fmt.Errorf("boom: not a valid .EXMODZ") +} + +// TestDownloadMod_DeployCompile_ValidatesAndRetainsNoPerModPak proves the +// #197 merged-only ingest contract: a downloaded .exmodz is validated and +// its bytes retained, but no per-mod pak is compiled or deployed - the +// merged pak (a separate, profile-level cache entry) is what actually +// deploys. +func TestDownloadMod_DeployCompile_ValidatesAndRetainsNoPerModPak(t *testing.T) { dlSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("fake-exmodz-bytes")) })) @@ -108,31 +140,28 @@ func TestDownloadMod_DeployCompile_InvokesCompiler(t *testing.T) { result, err := svc.DownloadMod(context.Background(), "fake-compiler", game, mod, file, nil) require.NoError(t, err) - require.Equal(t, 1, result.FilesExtracted) - require.Equal(t, 1, src.compileCalls) + require.Equal(t, 1, src.validateCalls, "ingest must validate the .exmodz") + require.Equal(t, 0, src.compileCalls, "ingest must NOT compile a per-mod pak (#197: merged-only)") + require.Equal(t, 0, result.FilesExtracted, "a per-mod exmodz cache entry has no deployment members under the merged-only model") gameCache := svc.GetGameCache(game) - require.True(t, gameCache.Exists(game.ID, mod.SourceID, mod.ID, mod.Version)) files, err := gameCache.ListFiles(game.ID, mod.SourceID, mod.ID, mod.Version) require.NoError(t, err) - require.Len(t, files, 1) - require.Equal(t, "Bear_Mount_P.pak", files[0]) + require.Empty(t, files, "ListFiles must report zero deployment members - the retained source is reserved, not a member") - data, err := os.ReadFile(gameCache.GetFilePath(game.ID, mod.SourceID, mod.ID, mod.Version, files[0])) + retainedPath := gameCache.GetFilePath(game.ID, mod.SourceID, mod.ID, mod.Version, cache.RetainedSourceName(file.ID)) + data, err := os.ReadFile(retainedPath) require.NoError(t, err) - require.Equal(t, "fake-exmodz-bytes", string(data)) + require.Equal(t, "fake-exmodz-bytes", string(data), "the original .exmodz bytes must still be retained") } -// newCompileTestGame builds a DeployCompile game backed by fakeCompilerSource, -// serving dlBody for every download - shared setup for -// TestDownloadMod_DeployCompile_RoutesPerFile's cases. -func newCompileTestGame(t *testing.T, dlBody string) (*core.Service, *fakeCompilerSource, *domain.Game) { - t.Helper() - +// TestDownloadMod_DeployCompile_MalformedExmodz_FailsLoudAtIngest proves +// validation happens at ingest time, not deferred to the next merge. +func TestDownloadMod_DeployCompile_MalformedExmodz_FailsLoudAtIngest(t *testing.T) { dlSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write([]byte(dlBody)) + _, _ = w.Write([]byte("not-a-valid-exmodz")) })) - t.Cleanup(dlSrv.Close) + defer dlSrv.Close() installDir := t.TempDir() basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") @@ -144,123 +173,18 @@ func newCompileTestGame(t *testing.T, dlBody string) (*core.Service, *fakeCompil require.NoError(t, err) t.Cleanup(func() { require.NoError(t, svc.Close()) }) - src := &fakeCompilerSource{downloadURL: dlSrv.URL} + src := &failingValidateCompilerSource{fakeCompilerSource: &fakeCompilerSource{downloadURL: dlSrv.URL}} svc.RegisterSource(src) game := &domain.Game{ID: "icarus", InstallPath: installDir, ModPath: t.TempDir(), DeployMode: domain.DeployCompile} require.NoError(t, svc.AddGame(game)) - return svc, src, game -} - -// TestDownloadMod_DeployCompile_RoutesPerFile pins the fix-round-1 gap: a -// DeployCompile game's compile branch must key off the FILE (".exmodz" -// suffix, case-insensitive), not the game alone - icarus.GetModFiles can -// serve a mod's already-built ".pak" alongside its ".exmodz" diff, and a pak -// routed into Compile fails (it isn't a zip ParseExmodz can read). -func TestDownloadMod_DeployCompile_RoutesPerFile(t *testing.T) { - tests := []struct { - name string - fileName string - wantCompiled bool - wantCachedName string - }{ - {"exmodz file takes the compile branch", "Bear_Mount.exmodz", true, "Bear_Mount_P.pak"}, - {"EXMODZ file takes the compile branch case-insensitively", "Bear_Mount.EXMODZ", true, "Bear_Mount_P.pak"}, - {"pak file skips the compiler entirely", "Bear_Mount.pak", false, "Bear_Mount.pak"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - const body = "fake-download-bytes" - svc, src, game := newCompileTestGame(t, body) - - mod := &domain.Mod{ID: "bear-mount", SourceID: "fake-compiler", GameID: "icarus", Version: "3.3"} - file := &domain.DownloadableFile{ID: "the-file", FileName: tt.fileName} - - result, err := svc.DownloadMod(context.Background(), "fake-compiler", game, mod, file, nil) - require.NoError(t, err) - require.Equal(t, 1, result.FilesExtracted) + mod := &domain.Mod{ID: "bad-mount", SourceID: "fake-compiler", GameID: "icarus", Version: "1.0"} + file := &domain.DownloadableFile{ID: "exmodz", FileName: "Bad_Mount.exmodz"} - wantCompileCalls := 0 - if tt.wantCompiled { - wantCompileCalls = 1 - } - require.Equal(t, wantCompileCalls, src.compileCalls) - - gameCache := svc.GetGameCache(game) - files, err := gameCache.ListFiles(game.ID, mod.SourceID, mod.ID, mod.Version) - require.NoError(t, err) - require.Equal(t, []string{tt.wantCachedName}, files) - - data, err := os.ReadFile(gameCache.GetFilePath(game.ID, mod.SourceID, mod.ID, mod.Version, tt.wantCachedName)) - require.NoError(t, err) - require.Equal(t, body, string(data)) - }) - } - - // Regression proof for the ".pak" case above: rather than only asserting - // "Compile wasn't called" (a fake-tautology that would also pass if - // routing were broken some other way), this proves a DeployCompile game - // handling a plain ".pak" produces EXACTLY what a DeployExtract game - // produces for the identical file through the identical source - the - // genuine pre-Task-13 extract/copy path, byte-for-byte. - t.Run("pak file on a compile-mode game matches a non-compile game byte-for-byte", func(t *testing.T) { - const body = "fake-pak-bytes" - mod := &domain.Mod{ID: "bear-mount", SourceID: "fake-compiler", GameID: "icarus", Version: "3.3"} - file := &domain.DownloadableFile{ID: "pak", FileName: "Bear_Mount.pak"} - - compileSvc, compileSrc, compileGame := newCompileTestGame(t, body) - compileResult, err := compileSvc.DownloadMod(context.Background(), "fake-compiler", compileGame, mod, file, nil) - require.NoError(t, err) - - extractSvc, extractSrc, extractGame := newCompileTestGame(t, body) - extractGame.DeployMode = domain.DeployExtract - extractResult, err := extractSvc.DownloadMod(context.Background(), "fake-compiler", extractGame, mod, file, nil) - require.NoError(t, err) - - require.Equal(t, 0, compileSrc.compileCalls) - require.Equal(t, 0, extractSrc.compileCalls) - require.Equal(t, extractResult, compileResult) - - compileFiles, err := compileSvc.GetGameCache(compileGame).ListFiles(compileGame.ID, mod.SourceID, mod.ID, mod.Version) - require.NoError(t, err) - extractFiles, err := extractSvc.GetGameCache(extractGame).ListFiles(extractGame.ID, mod.SourceID, mod.ID, mod.Version) - require.NoError(t, err) - require.Equal(t, extractFiles, compileFiles) - - compileData, err := os.ReadFile(compileSvc.GetGameCache(compileGame).GetFilePath(compileGame.ID, mod.SourceID, mod.ID, mod.Version, compileFiles[0])) - require.NoError(t, err) - extractData, err := os.ReadFile(extractSvc.GetGameCache(extractGame).GetFilePath(extractGame.ID, mod.SourceID, mod.ID, mod.Version, extractFiles[0])) - require.NoError(t, err) - require.Equal(t, extractData, compileData) - }) -} - -// TestDownloadMod_DeployCompile_MixedFileMod pins that a single mod shipping -// both a prebuilt ".pak" and an ".exmodz" diff (icarus.GetModFiles's "pak" -// then "exmodz" enumeration, neither marked primary when both are present) -// gets each file routed independently within the same DeployCompile game: -// one DownloadMod call per DownloadableFile, exactly as the real CLI/TUI -// download flow drives it. -func TestDownloadMod_DeployCompile_MixedFileMod(t *testing.T) { - svc, src, game := newCompileTestGame(t, "fake-bytes") - mod := &domain.Mod{ID: "bear-mount", SourceID: "fake-compiler", GameID: "icarus", Version: "3.3"} - - exmodzFile := &domain.DownloadableFile{ID: "exmodz", FileName: "Bear_Mount.exmodz"} - pakFile := &domain.DownloadableFile{ID: "pak", FileName: "Bear_Mount.pak"} - - _, err := svc.DownloadMod(context.Background(), "fake-compiler", game, mod, exmodzFile, nil) - require.NoError(t, err) - require.Equal(t, 1, src.compileCalls, "exmodz file must compile") - - _, err = svc.DownloadMod(context.Background(), "fake-compiler", game, mod, pakFile, nil) - require.NoError(t, err) - require.Equal(t, 1, src.compileCalls, "pak file must not trigger a second compile") + _, err = svc.DownloadMod(context.Background(), "fake-compiler", game, mod, file, nil) + require.Error(t, err) gameCache := svc.GetGameCache(game) - files, err := gameCache.ListFiles(game.ID, mod.SourceID, mod.ID, mod.Version) - require.NoError(t, err) - require.ElementsMatch(t, []string{"Bear_Mount_P.pak", "Bear_Mount.pak"}, files, - "both the compiled exmodz output and the untouched pak must be cached") + require.False(t, gameCache.Exists(game.ID, mod.SourceID, mod.ID, mod.Version), "a validation failure must leave no cache entry") } From b8f86381d2b326ad823ccb3781d0972407812cf6 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 20:41:35 -0400 Subject: [PATCH 67/96] feat: import path ingests .exmodz as validate+retain, no per-mod pak (#197) --- internal/core/importer.go | 74 ++--- internal/core/service_import_compile_test.go | 286 +++---------------- 2 files changed, 63 insertions(+), 297 deletions(-) diff --git a/internal/core/importer.go b/internal/core/importer.go index 392a6ca..f6e2dca 100644 --- a/internal/core/importer.go +++ b/internal/core/importer.go @@ -37,15 +37,15 @@ type Importer struct { // stagingRoot is where archives are extracted before being committed to the // cache. Empty means fall back to $TMPDIR — see newStagingDir. stagingRoot string - // resolveCompiler resolves the Compiler-capable source mapped to a - // DeployCompile game's registry entry (#173), consulted only when + // resolveMergeCompiler resolves the MergeCompiler-capable source mapped + // to a DeployCompile game's registry entry (#197), consulted only when // importing a ".exmodz" archive for such a game — Import has no // per-archive source pinned the way DownloadModToCache does, so it must // look up the game's configured sources instead. nil when the Importer // was built via the standalone NewImporter (no Service context): - // compiling an .exmodz through such an Importer fails loud rather than - // silently caching the uncompiled archive. - resolveCompiler func(gameID string) (source.Compiler, error) + // importing an .exmodz through such an Importer fails loud rather than + // silently caching an unvalidated archive. + resolveMergeCompiler func(gameID string) (source.MergeCompiler, error) } // NewImporter creates a new Importer that stages extraction in the OS temp dir. @@ -62,7 +62,7 @@ func NewImporter(cache *cache.Cache) *Importer { func (s *Service) NewImporter(game *domain.Game) *Importer { imp := NewImporter(s.GetGameCache(game)) imp.stagingRoot = s.stagingRoot() - imp.resolveCompiler = s.compilerSourceForGame + imp.resolveMergeCompiler = s.mergeCompilerSourceForGame return imp } @@ -108,23 +108,21 @@ func (i *Importer) Import(ctx context.Context, archivePath string, game *domain. // Handle based on game's deploy mode if game.DeployMode == domain.DeployCompile && isExmodzFile(filename) { - // Compile mode (#173): mirror Service.DownloadModToCache's - // DeployCompile branch — compile the archive against the game's - // installed base pak instead of extracting or copying it verbatim, - // caching the compiled *_P.pak the same way a downloaded .exmodz - // would be. Unlike the download path, Import has no per-archive - // source pinned to check for source.Compiler, so it resolves the - // game's mapped compiler-capable source from the registry instead. - if i.resolveCompiler == nil { + // Validate mode (#197): Import has no real source file ID the way a + // download does (DownloadableFile.ID is resolved later, outside + // Import, only when --id was given), so the retained source is + // keyed by the archive's own filename instead - stable across + // re-imports of the same name, and the ONLY identity Import ever + // has for this content. + if i.resolveMergeCompiler == nil { return nil, fmt.Errorf("game %q requires DeployCompile to import %q, but this Importer was constructed without service context (via core.NewImporter, not Service.NewImporter) and has no compiler resolver to consult - import via the service-backed importer instead", game.ID, filename) } - compiler, err := i.resolveCompiler(game.ID) + mc, err := i.resolveMergeCompiler(game.ID) if err != nil { return nil, err } - basePakPath, err := resolveBasePak(game) - if err != nil { - return nil, err + if err := mc.ValidateSource(archivePath); err != nil { + return nil, fmt.Errorf("validating %s: %w", filename, err) } modName = strings.TrimSuffix(filename, filepath.Ext(filename)) @@ -134,28 +132,6 @@ func (i *Importer) Import(ctx context.Context, archivePath string, game *domain. } } - // Compile into a scratch dir first so a mid-compile failure never - // leaves a partial/uncompiled artifact in the cache (mirrors #136 - // review's fix for the download path's compile branch). - tempDir, err := newStagingDir(i.stagingRoot, "lmm-import-compile-*") - if err != nil { - return nil, err - } - defer os.RemoveAll(tempDir) //nolint:errcheck - - destName := compiledFileName(filename) - compiledPath := filepath.Join(tempDir, destName) - if err := compiler.Compile(ctx, basePakPath, archivePath, compiledPath); err != nil { - return nil, fmt.Errorf("compiling mod: %w", err) - } - - // Stage the compiled artifact and commit it atomically (#173 review: - // mirrors Service.DownloadModToCache's compile branch). cachePath is - // never touched until commitStagedCache's single backup-then-rename - // swap, so a failure staging the artifact (or committing it) leaves - // any existing cache entry exactly as it was — unlike the previous - // remove-existing-then-copy sequence, which destroyed the prior - // entry before the copy that could still fail. cacheMod := &domain.Mod{ID: modID, SourceID: sourceID, Version: version, GameID: game.ID} cachePath, stagePath, err := prepareUnseededStaging(i.cache, game, cacheMod) if err != nil { @@ -166,24 +142,14 @@ func (i *Importer) Import(ctx context.Context, archivePath string, game *domain. if err := os.MkdirAll(stagePath, 0755); err != nil { return nil, fmt.Errorf("preparing cache staging: %w", err) } - if err := copyFileStreaming(compiledPath, filepath.Join(stagePath, destName)); err != nil { - return nil, fmt.Errorf("staging compiled mod: %w", err) - } - // #196: stage the compile fingerprint (base pak IndexHash) and - // retained source keyed by destName - Import has no real source - // file ID the way a download does (DownloadableFile.ID is resolved - // later, outside Import, only when --id was given), so the compiled - // output's own filename is the stable per-entry key instead. A - // re-import of the same archive name replaces this entry outright - // (prepareUnseededStaging), so destName never collides across - // generations of the same mod. - if err := stageCompileFingerprint(stagePath, destName, basePakPath, archivePath); err != nil { - return nil, err + retainedPath := filepath.Join(stagePath, cache.RetainedSourceName(filename)) + if err := copyFileStreaming(archivePath, retainedPath); err != nil { + return nil, fmt.Errorf("retaining %s: %w", filename, err) } if err := commitStagedCache(cachePath, stagePath); err != nil { return nil, err } - fileCount = 1 + fileCount = 0 } else if game.DeployMode == domain.DeployCopy { // Copy mode: just copy the file as-is to cache (don't extract) modName = strings.TrimSuffix(filename, filepath.Ext(filename)) diff --git a/internal/core/service_import_compile_test.go b/internal/core/service_import_compile_test.go index 6f32564..59f6cbd 100644 --- a/internal/core/service_import_compile_test.go +++ b/internal/core/service_import_compile_test.go @@ -2,7 +2,6 @@ package core_test import ( "context" - "fmt" "os" "path/filepath" "testing" @@ -13,48 +12,12 @@ import ( "github.com/stretchr/testify/require" ) -// failingCompilerSource wraps fakeCompilerSource (defined in -// service_icarus_compile_test.go) and shadows Compile to always fail, -// letting failure-leg tests below prove a mid-compile error never leaves a -// partial artifact behind (#173, mirroring #136 review's "remove partial -// output pak on mid-compile failure" fix for the download path). -type failingCompilerSource struct { - *fakeCompilerSource -} - -func (s *failingCompilerSource) Compile(ctx context.Context, basePakPath, sourceFilePath, outputPath string) error { - return fmt.Errorf("boom: compile always fails") -} - -// raceCompilerSource wraps fakeCompilerSource and, when sabotage is set, -// writes its declared output and then immediately removes it before -// returning success - deterministically reproducing "compile reported -// success, but the artifact is gone by the time it must be staged into the -// cache" without any OS-specific permission tricks. This is the shape of -// the #173 review defect: the compile step itself succeeds, but the -// subsequent step that gets the artifact into the cache can still fail. -type raceCompilerSource struct { - *fakeCompilerSource - sabotage bool -} - -func (s *raceCompilerSource) Compile(ctx context.Context, basePakPath, sourceFilePath, outputPath string) error { - s.compileCalls++ - if err := os.WriteFile(outputPath, []byte("new-content"), 0o644); err != nil { - return err - } - if s.sabotage { - return os.Remove(outputPath) - } - return nil -} - // newImportCompileTestGame builds a DeployCompile game with a registered, -// game-mapped compiler source and an installed base pak - the setup #173's -// import path needs to resolve a Compiler the same way +// game-mapped merge-compiler source and an installed base pak - the setup +// #173/#197's import path needs to resolve a MergeCompiler the same way // Service.DownloadModToCache resolves one from the download's own source, // except import has no per-download source pinned, so it must resolve the -// compiler from the game's registered sources instead (game.SourceIDs). +// merge-compiler from the game's registered sources instead (game.SourceIDs). func newImportCompileTestGame(t *testing.T) (*core.Service, *fakeCompilerSource, *domain.Game) { t.Helper() @@ -83,7 +46,11 @@ func newImportCompileTestGame(t *testing.T) (*core.Service, *fakeCompilerSource, return svc, src, game } -func TestImportMod_DeployCompile_ExmodzCompiles(t *testing.T) { +// TestImportMod_DeployCompile_ValidatesAndRetainsNoPerModPak proves the +// #197 merged-only ingest contract for the import path: an imported +// .exmodz is validated and its bytes retained, but no per-mod pak is +// compiled - matching the download path's behavior exactly. +func TestImportMod_DeployCompile_ValidatesAndRetainsNoPerModPak(t *testing.T) { svc, src, game := newImportCompileTestGame(t) tempDir := t.TempDir() @@ -93,78 +60,51 @@ func TestImportMod_DeployCompile_ExmodzCompiles(t *testing.T) { importer := svc.NewImporter(game) result, err := importer.Import(context.Background(), archivePath, game, core.ImportOptions{}) require.NoError(t, err) - require.Equal(t, 1, result.FilesExtracted) - require.Equal(t, 1, src.compileCalls) + require.Equal(t, 1, src.validateCalls) + require.Equal(t, 0, src.compileCalls, "import must NOT compile a per-mod pak (#197: merged-only)") + require.Equal(t, 0, result.FilesExtracted) gameCache := svc.GetGameCache(game) files, err := gameCache.ListFiles(game.ID, result.Mod.SourceID, result.Mod.ID, result.Mod.Version) require.NoError(t, err) - require.Equal(t, []string{"Bear_Mount_P.pak"}, files) + require.Empty(t, files) - data, err := os.ReadFile(gameCache.GetFilePath(game.ID, result.Mod.SourceID, result.Mod.ID, result.Mod.Version, files[0])) + // Import has no real DownloadableFile.ID (see the field's own doc + // comment) - it keys the retained source by the ARCHIVE'S OWN filename + // instead, exactly as the #196-era destName-keying did. + retainedPath := gameCache.GetFilePath(game.ID, result.Mod.SourceID, result.Mod.ID, result.Mod.Version, cache.RetainedSourceName("Bear_Mount.exmodz")) + data, err := os.ReadFile(retainedPath) require.NoError(t, err) require.Equal(t, "fake-exmodz-bytes", string(data)) } -// TestImportMod_DeployCompile_RoutesPerFile mirrors -// TestDownloadMod_DeployCompile_RoutesPerFile (service_icarus_compile_test.go): -// only a ".exmodz" suffix (case-insensitive) takes the compile branch. A -// plain ".pak" import is untouched by #173 - it falls through to the -// existing extract-mode branch exactly as it did before this change, which -// today means "unsupported archive format" (pak isn't a recognized -// archive), pinned here as a regression proof that non-exmodz import -// behavior is unchanged. -func TestImportMod_DeployCompile_RoutesPerFile(t *testing.T) { - tests := []struct { - name string - fileName string - wantCompiled bool - wantCachedName string - wantErrContains string - }{ - {name: "exmodz file takes the compile branch", fileName: "Bear_Mount.exmodz", wantCompiled: true, wantCachedName: "Bear_Mount_P.pak"}, - {name: "EXMODZ file takes the compile branch case-insensitively", fileName: "Bear_Mount.EXMODZ", wantCompiled: true, wantCachedName: "Bear_Mount_P.pak"}, - {name: "pak file is unaffected: today's unsupported-archive error is unchanged", fileName: "Bear_Mount.pak", wantErrContains: "unsupported archive format"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - svc, src, game := newImportCompileTestGame(t) - - tempDir := t.TempDir() - archivePath := filepath.Join(tempDir, tt.fileName) - require.NoError(t, os.WriteFile(archivePath, []byte("fake-bytes"), 0o644)) - - importer := svc.NewImporter(game) - result, err := importer.Import(context.Background(), archivePath, game, core.ImportOptions{}) +// TestImportMod_DeployCompile_MalformedExmodz_FailsLoud proves validation +// happens at import time - the only failure mode left in this branch since +// there is no per-mod compile step anymore. +func TestImportMod_DeployCompile_MalformedExmodz_FailsLoud(t *testing.T) { + svc, src, game := newImportCompileTestGame(t) + _ = src // validation failure is injected by wrapping, not by this fake - if tt.wantErrContains != "" { - require.Error(t, err) - require.Contains(t, err.Error(), tt.wantErrContains) - require.Equal(t, 0, src.compileCalls) - return - } + failing := &failingValidateCompilerSource{fakeCompilerSource: &fakeCompilerSource{}} + // Re-register under the same source ID so the importer resolves the + // failing wrapper instead of the passing fake newImportCompileTestGame + // already registered. + svc.RegisterSource(failing) - require.NoError(t, err) - wantCompileCalls := 0 - if tt.wantCompiled { - wantCompileCalls = 1 - } - require.Equal(t, wantCompileCalls, src.compileCalls) + tempDir := t.TempDir() + archivePath := filepath.Join(tempDir, "Bad_Mount.exmodz") + require.NoError(t, os.WriteFile(archivePath, []byte("not-a-valid-exmodz"), 0o644)) - gameCache := svc.GetGameCache(game) - files, err := gameCache.ListFiles(game.ID, result.Mod.SourceID, result.Mod.ID, result.Mod.Version) - require.NoError(t, err) - require.Equal(t, []string{tt.wantCachedName}, files) - }) - } + importer := svc.NewImporter(game) + _, err := importer.Import(context.Background(), archivePath, game, core.ImportOptions{}) + require.Error(t, err) } // TestImportMod_DeployCompile_ZipPassthroughUnaffected proves a regular // (non-exmodz) archive import for a DeployCompile game is byte-for-byte -// identical to the same import against a DeployExtract game - #173 only -// inserts a new leading branch keyed on isExmodzFile, it must never change -// behavior for anything else. +// identical to the same import against a DeployExtract game - #173/#197 +// only insert a new leading branch keyed on isExmodzFile, it must never +// change behavior for anything else. func TestImportMod_DeployCompile_ZipPassthroughUnaffected(t *testing.T) { makeArchive := func(t *testing.T) string { t.Helper() @@ -192,9 +132,10 @@ func TestImportMod_DeployCompile_ZipPassthroughUnaffected(t *testing.T) { } // TestImportMod_DeployCompile_NoCompilerSourceFailsLoud pins the "never -// silently cache an uncompiled .exmodz" requirement (#173): a DeployCompile -// game with no Compiler-capable source mapped in its SourceIDs must fail -// loud with an actionable error instead of falling through to extract/copy. +// silently cache an unvalidated .exmodz" requirement (#173/#197): a +// DeployCompile game with no MergeCompiler-capable source mapped in its +// SourceIDs must fail loud with an actionable error instead of falling +// through to extract/copy. func TestImportMod_DeployCompile_NoCompilerSourceFailsLoud(t *testing.T) { installDir := t.TempDir() basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") @@ -207,7 +148,7 @@ func TestImportMod_DeployCompile_NoCompilerSourceFailsLoud(t *testing.T) { t.Cleanup(func() { require.NoError(t, svc.Close()) }) // No RegisterSource call at all - the game has no source mapped, let - // alone a Compiler-capable one. + // alone a MergeCompiler-capable one. game := &domain.Game{ID: "icarus", InstallPath: installDir, ModPath: t.TempDir(), DeployMode: domain.DeployCompile} require.NoError(t, svc.AddGame(game)) @@ -225,86 +166,10 @@ func TestImportMod_DeployCompile_NoCompilerSourceFailsLoud(t *testing.T) { require.True(t, os.IsNotExist(statErr), "no cache entry should have been created") } -// TestImportMod_DeployCompile_MissingBasePakFailsLoud pins the second -// "fail loud when compilation is impossible" leg (#173): a game whose -// installed base pak is missing must error instead of compiling against -// nothing or silently caching the raw archive. -func TestImportMod_DeployCompile_MissingBasePakFailsLoud(t *testing.T) { - installDir := t.TempDir() // no Icarus/Content/Data/data.pak written - - cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} - svc, err := core.NewService(cfg) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, svc.Close()) }) - - src := &fakeCompilerSource{} - svc.RegisterSource(src) - - game := &domain.Game{ - ID: "icarus", - InstallPath: installDir, - ModPath: t.TempDir(), - DeployMode: domain.DeployCompile, - SourceIDs: map[string]string{"fake-compiler": "external-icarus-id"}, - } - require.NoError(t, svc.AddGame(game)) - - tempDir := t.TempDir() - archivePath := filepath.Join(tempDir, "Bear_Mount.exmodz") - require.NoError(t, os.WriteFile(archivePath, []byte("fake-exmodz-bytes"), 0o644)) - - importer := svc.NewImporter(game) - result, err := importer.Import(context.Background(), archivePath, game, core.ImportOptions{}) - require.Error(t, err) - require.Nil(t, result) - require.Contains(t, err.Error(), "base pak") - require.Equal(t, 0, src.compileCalls) -} - -// TestImportMod_DeployCompile_CompileFailureLeavesNoPartialArtifact proves a -// mid-compile failure never lands a partial/uncompiled file in the cache -// (#173 - "never silently cache an uncompiled .exmodz"). -func TestImportMod_DeployCompile_CompileFailureLeavesNoPartialArtifact(t *testing.T) { - installDir := t.TempDir() - basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") - require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) - writeFakeBasePak(t, basePak) - - cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} - svc, err := core.NewService(cfg) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, svc.Close()) }) - - src := &failingCompilerSource{fakeCompilerSource: &fakeCompilerSource{}} - svc.RegisterSource(src) - - game := &domain.Game{ - ID: "icarus", - InstallPath: installDir, - ModPath: t.TempDir(), - DeployMode: domain.DeployCompile, - SourceIDs: map[string]string{"fake-compiler": "external-icarus-id"}, - } - require.NoError(t, svc.AddGame(game)) - - tempDir := t.TempDir() - archivePath := filepath.Join(tempDir, "Bear_Mount.exmodz") - require.NoError(t, os.WriteFile(archivePath, []byte("fake-exmodz-bytes"), 0o644)) - - importer := svc.NewImporter(game) - result, err := importer.Import(context.Background(), archivePath, game, core.ImportOptions{}) - require.Error(t, err) - require.Nil(t, result) - require.Contains(t, err.Error(), "compiling mod") - - _, statErr := os.Stat(filepath.Join(cfg.CacheDir, game.ID)) - require.True(t, os.IsNotExist(statErr), "no partial cache entry should have been created") -} - // TestImportMod_DeployCompile_StandaloneImporterFailsLoud proves an // Importer constructed without Service context (core.NewImporter, used // directly in older tests) still fails loud rather than silently caching an -// uncompiled .exmodz - it simply has no compiler resolver to consult. +// unvalidated .exmodz - it simply has no compiler resolver to consult. func TestImportMod_DeployCompile_StandaloneImporterFailsLoud(t *testing.T) { tempDir := t.TempDir() cacheDir := filepath.Join(tempDir, "cache") @@ -324,68 +189,3 @@ func TestImportMod_DeployCompile_StandaloneImporterFailsLoud(t *testing.T) { require.Contains(t, err.Error(), "without service context") require.Contains(t, err.Error(), "core.NewImporter") } - -// TestImportMod_DeployCompile_ReimportSurvivesStagingFailure pins the #173 -// review defect: the compile branch used to os.RemoveAll(cachePath) and -// then copy the compiled artifact into place, so a failure in that copy -// step destroyed a pre-existing good cache entry before ever writing its -// replacement. It now stages the compiled artifact into an isolated -// directory and commits it to cachePath atomically (commitStagedCache, -// mirroring the download path) - cachePath is never touched by a failed -// staging attempt, so a pre-existing entry must survive. -func TestImportMod_DeployCompile_ReimportSurvivesStagingFailure(t *testing.T) { - installDir := t.TempDir() - basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") - require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) - writeFakeBasePak(t, basePak) - - cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} - svc, err := core.NewService(cfg) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, svc.Close()) }) - - src := &raceCompilerSource{fakeCompilerSource: &fakeCompilerSource{}} - svc.RegisterSource(src) - - game := &domain.Game{ - ID: "icarus", - InstallPath: installDir, - ModPath: t.TempDir(), - DeployMode: domain.DeployCompile, - SourceIDs: map[string]string{"fake-compiler": "external-icarus-id"}, - } - require.NoError(t, svc.AddGame(game)) - - tempDir := t.TempDir() - archivePath := filepath.Join(tempDir, "Bear_Mount.exmodz") - require.NoError(t, os.WriteFile(archivePath, []byte("good-exmodz-bytes"), 0o644)) - - opts := core.ImportOptions{SourceID: "fake-compiler", ModID: "bear-mount"} - importer := svc.NewImporter(game) - - // First import succeeds and leaves a good cache entry. - result1, err := importer.Import(context.Background(), archivePath, game, opts) - require.NoError(t, err) - - gameCache := svc.GetGameCache(game) - files, err := gameCache.ListFiles(game.ID, result1.Mod.SourceID, result1.Mod.ID, result1.Mod.Version) - require.NoError(t, err) - require.Equal(t, []string{"Bear_Mount_P.pak"}, files) - - filePath := gameCache.GetFilePath(game.ID, result1.Mod.SourceID, result1.Mod.ID, result1.Mod.Version, files[0]) - origData, err := os.ReadFile(filePath) - require.NoError(t, err) - require.Equal(t, "new-content", string(origData)) - - // Re-import the same archive; the compiler's declared output vanishes - // before it can be staged, so this import must fail... - src.sabotage = true - result2, err := importer.Import(context.Background(), archivePath, game, opts) - require.Error(t, err) - require.Nil(t, result2) - - // ...and the prior good entry must still be exactly what it was. - survivingData, err := os.ReadFile(filePath) - require.NoError(t, err, "the prior good cache entry must survive a failed re-import") - require.Equal(t, "new-content", string(survivingData)) -} From 24cd38ac7626c35a805cf5fe273057876623fd9d Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 20:45:18 -0400 Subject: [PATCH 68/96] feat: MergedFingerprint type + Service.syncMergedPak regenerate engine (#197) --- internal/core/merged_pak.go | 296 +++++++++++++++++++ internal/core/merged_pak_internal_test.go | 108 +++++++ internal/core/merged_pak_test.go | 252 ++++++++++++++++ internal/core/service_icarus_compile_test.go | 16 +- internal/domain/mod.go | 6 + internal/storage/cache/cache.go | 16 + internal/storage/cache/cache_test.go | 24 ++ 7 files changed, 717 insertions(+), 1 deletion(-) create mode 100644 internal/core/merged_pak.go create mode 100644 internal/core/merged_pak_internal_test.go create mode 100644 internal/core/merged_pak_test.go diff --git a/internal/core/merged_pak.go b/internal/core/merged_pak.go new file mode 100644 index 0000000..7345d88 --- /dev/null +++ b/internal/core/merged_pak.go @@ -0,0 +1,296 @@ +package core + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/source" + "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" +) + +// mergedPakModID/mergedPakVersion/mergedPakFileName identify the merged pak +// as a synthetic, singleton "mod" per (game, profile) - domain.SourceMerged +// is the matching sourceID. This reuses Installer.Install/Uninstall and +// cache.Cache verbatim (#197 design decision 2) rather than a parallel +// deploy/tracking mechanism: zero schema changes, and the SAME +// deployed_files ownership (and #168-class residue risk) as every other +// deployed file. +const ( + mergedPakModID = "merged-pak" + // mergedPakVersion is fixed ("merged", not a real upstream version) - + // there is exactly one merged pak per (game, profile) at any time, and + // every regeneration REPLACES it outright (mirrors #166's directory- + // source "replace, don't overlay" precedent) rather than versioning it. + mergedPakVersion = "merged" + // mergedPakFileName sorts LAST among files UE mounts from a profile's + // mods directory: paks mount in filename-sort order and a later mount + // wins same-path conflicts (this repo's own icarusContentMountPoint doc + // comment, and #197's issue body, both note this) - "zzz" is a + // long-standing UE-modding convention for "load last, highest + // priority", so the merged pak's authoritative combined table state can + // never be silently shadowed by a plain prebuilt .pak mod that happens + // to also carry a table override. "LMM" makes the file greppable as + // lmm-owned; "_P" matches this codebase's existing override-pak suffix + // convention (compiledFileName). + mergedPakFileName = "zzz_LMM_Merged_P.pak" +) + +// MergedFingerprint captures everything a merged pak was built from (#197): +// the base pak's IndexHash plus an ORDERED list of every contributing +// exmodz file's identity and content checksum. Order matters - it's the +// profile's load order, which is also merge-application order - so two +// fingerprints with the same entries in a DIFFERENT order must compare +// unequal (a load-order change is a documented regeneration trigger). +type MergedFingerprint struct { + BaseIndexHash string + Mods []MergedFingerprintEntry +} + +// MergedFingerprintEntry identifies one contributing file within a +// MergedFingerprint. +type MergedFingerprintEntry struct { + SourceID string + ModID string + Version string + Checksum string // MD5 of the retained .exmodz bytes (md5File) +} + +// marshalMergedFingerprint renders f deterministically: encoding/json +// marshals struct fields in declaration order (not sorted) and preserves +// slice order exactly, so the same MergedFingerprint value always produces +// byte-identical output - the property mergedFingerprintsEqual depends on. +// +// A nil Mods is normalized to an empty (non-nil) slice first: encoding/json +// marshals a nil slice as `null` but an empty slice as `[]` - two DIFFERENT +// byte sequences for what must count as the same "zero contributing mods" +// state (e.g. a freshly-built "current" fingerprint via `var mods []T` +// compared against a previously-stored marker written some other way). +// Caught by extraction-verification (a scratch test comparing the two +// literally failed before this normalization was added) - without it, a +// profile with zero enabled exmodz mods could spuriously flip between +// "stale"/"not stale" depending on which code path happened to build each +// side's slice. +func marshalMergedFingerprint(f MergedFingerprint) ([]byte, error) { + if f.Mods == nil { + f.Mods = []MergedFingerprintEntry{} + } + return json.Marshal(f) +} + +// mergedFingerprintsEqual reports whether a and b describe the same merge +// inputs, by comparing their marshaled bytes - exactly what "compare +// against the stored marker" needs, since the marker itself IS the +// marshaled form. +func mergedFingerprintsEqual(a, b MergedFingerprint) (bool, error) { + aBytes, err := marshalMergedFingerprint(a) + if err != nil { + return false, err + } + bBytes, err := marshalMergedFingerprint(b) + if err != nil { + return false, err + } + return bytes.Equal(aBytes, bBytes), nil +} + +// enabledExmodzSources returns every enabled mod's retained .exmodz files +// for game+profileName, in PROFILE LOAD ORDER (the merge-application order, +// #197 design) - the exact input MergeCompile needs. Only files that were +// actually retained (cache.RetainedSourceName present in the mod's cache +// entry) count: a mod's plain .pak files, or a mod whose ingest never got +// far enough to retain anything, contribute nothing. A mod's OWN FileIDs +// are walked (not the whole cache directory) because a download-compiled +// entry's retained-source name is keyed by a real DownloadableFile.ID, +// while an import-compiled entry's is keyed by its own archive filename +// (see Task 2/3's ingest branches) - FileIDs is the one list that already +// carries whichever identity applies, for either origin. +func (s *Service) enabledExmodzSources(game *domain.Game, profileName string) ([]source.MergeSource, error) { + mods, err := s.GetInstalledModsInProfileOrder(game.ID, profileName) + if err != nil { + return nil, fmt.Errorf("loading profile mods: %w", err) + } + + gameCache := s.GetGameCache(game) + var sources []source.MergeSource + for _, mod := range mods { + if !mod.Enabled { + continue + } + for _, fileID := range mod.FileIDs { + retainedPath := gameCache.GetFilePath(game.ID, mod.SourceID, mod.ID, mod.Version, cache.RetainedSourceName(fileID)) + if _, statErr := os.Stat(retainedPath); statErr != nil { + continue // not a retained exmodz file (a plain .pak's fileID, or nothing ingested) + } + sources = append(sources, source.MergeSource{ + ModRef: mod.SourceID + ":" + mod.ID, + ExmodzPath: retainedPath, + }) + } + } + return sources, nil +} + +// EnabledExmodzSourcesForTest exposes enabledExmodzSources to external +// (core_test package) tests - the method itself stays unexported since it +// is an internal implementation detail of syncMergedPak, not part of +// Service's public API. +func (s *Service) EnabledExmodzSourcesForTest(game *domain.Game, profileName string) ([]source.MergeSource, error) { + return s.enabledExmodzSources(game, profileName) +} + +// syncMergedPak regenerates game+profileName's merged pak if its recorded +// fingerprint no longer matches the CURRENT enabled-mod set/order/versions/ +// base pak (#197). Cheap when nothing changed: the fast path is one +// directory read (enabledExmodzSources), one base-pak footer read +// (basePakIndexHash - never the pak's full content), and N small MD5s +// (md5File over each retained .exmodz - real files here are small, see +// #175's own research on real base-table sizes), then a byte comparison. +// Safe to call unconditionally from ANY mutation flow regardless of game +// type - it no-ops immediately for a non-DeployCompile game. +// +// Zero enabled exmodz sources uninstalls any existing merged pak instead of +// generating an empty one (#197 design decision 2's "uninstall-to-zero" +// requirement) - Installer.Uninstall on the synthetic merged-pak mod is +// idempotent when there is nothing deployed (linker.Undeploy tolerates an +// already-absent path, matching every other uninstall in this codebase), +// so calling it unconditionally here is safe even when no pak was ever +// generated. +func (s *Service) syncMergedPak(ctx context.Context, game *domain.Game, profileName string) (warnings []string, err error) { + if game.DeployMode != domain.DeployCompile { + return nil, nil + } + + sources, err := s.enabledExmodzSources(game, profileName) + if err != nil { + return nil, fmt.Errorf("listing enabled exmodz mods: %w", err) + } + + gameCache := s.GetGameCache(game) + syntheticMod := &domain.Mod{ID: mergedPakModID, SourceID: domain.SourceMerged, Version: mergedPakVersion, GameID: game.ID} + + installer, err := s.GetInstallerForProfile(game, profileName) + if err != nil { + return nil, err + } + + if len(sources) == 0 { + if uerr := installer.Uninstall(ctx, game, syntheticMod, profileName); uerr != nil { + return nil, fmt.Errorf("removing merged pak: %w", uerr) + } + if derr := gameCache.Delete(game.ID, domain.SourceMerged, mergedPakModID, mergedPakVersion); derr != nil { + return nil, fmt.Errorf("clearing merged pak cache entry: %w", derr) + } + return nil, nil + } + + basePakPath, err := resolveBasePak(game) + if err != nil { + return nil, err + } + liveHash, err := basePakIndexHash(basePakPath) + if err != nil { + return nil, fmt.Errorf("reading base pak for merge fingerprint: %w", err) + } + + current := MergedFingerprint{BaseIndexHash: liveHash} + for _, src := range sources { + sum, herr := md5File(src.ExmodzPath) + if herr != nil { + return nil, fmt.Errorf("hashing %s: %w", src.ExmodzPath, herr) + } + sourceID, modID, _ := strings.Cut(src.ModRef, ":") + current.Mods = append(current.Mods, MergedFingerprintEntry{ + SourceID: sourceID, ModID: modID, Checksum: sum, + }) + } + // Version is not carried on source.MergeSource (it only needs ModRef + + // ExmodzPath for the merge itself) - resolved separately here so + // enabledExmodzSources' own signature stays minimal. Re-fetching the + // installed mods once more is cheap (small profiles) and keeps + // enabledExmodzSources' contract focused on ONE job. + mods, err := s.GetInstalledModsInProfileOrder(game.ID, profileName) + if err != nil { + return nil, fmt.Errorf("loading profile mods: %w", err) + } + versionByRef := make(map[string]string, len(mods)) + for _, m := range mods { + versionByRef[m.SourceID+":"+m.ID] = m.Version + } + for i, src := range sources { + current.Mods[i].Version = versionByRef[src.ModRef] + } + + cachePath := gameCache.ModPath(game.ID, domain.SourceMerged, mergedPakModID, mergedPakVersion) + if stored, ok := readMergedFingerprint(cachePath); ok { + if eq, eqErr := mergedFingerprintsEqual(current, stored); eqErr == nil && eq { + return nil, nil // fast path: nothing changed + } + } + + mc, err := s.mergeCompilerSourceForGame(game.ID) + if err != nil { + return nil, err + } + + stagePath := cachePath + ".staging" + if err := os.RemoveAll(stagePath); err != nil { + return nil, fmt.Errorf("clearing merged pak staging: %w", err) + } + if err := os.MkdirAll(stagePath, 0755); err != nil { + return nil, fmt.Errorf("preparing merged pak staging: %w", err) + } + defer os.RemoveAll(stagePath) //nolint:errcheck + + outputPath := filepath.Join(stagePath, mergedPakFileName) + mergeWarnings, err := mc.MergeCompile(ctx, basePakPath, sources, outputPath) + if err != nil { + return nil, fmt.Errorf("merging %d exmodz mod(s): %w", len(sources), err) + } + warnings = mergeWarnings + + fingerprintBytes, err := marshalMergedFingerprint(current) + if err != nil { + return warnings, fmt.Errorf("encoding merge fingerprint: %w", err) + } + if err := os.WriteFile(cache.MergeFingerprintPath(stagePath), fingerprintBytes, 0644); err != nil { + return warnings, fmt.Errorf("writing merge fingerprint: %w", err) + } + + if err := commitStagedCache(cachePath, stagePath); err != nil { + return warnings, err + } + + if err := installer.Install(ctx, game, syntheticMod, profileName); err != nil { + return warnings, fmt.Errorf("deploying merged pak: %w", err) + } + return warnings, nil +} + +// SyncMergedPakForTest exposes syncMergedPak to external (core_test +// package) tests - see enabledExmodzSources/EnabledExmodzSourcesForTest's +// identical rationale. +func (s *Service) SyncMergedPakForTest(ctx context.Context, game *domain.Game, profileName string) ([]string, error) { + return s.syncMergedPak(ctx, game, profileName) +} + +// readMergedFingerprint reads and decodes cachePath's stored merge +// fingerprint marker, if any. ok is false when no cache entry/marker +// exists yet (first-ever merge for this profile) or the marker is +// unreadable/corrupt - both degrade to "regenerate", never a crash or a +// false "unchanged". +func readMergedFingerprint(cachePath string) (fp MergedFingerprint, ok bool) { + data, err := os.ReadFile(cache.MergeFingerprintPath(cachePath)) + if err != nil { + return MergedFingerprint{}, false + } + if err := json.Unmarshal(data, &fp); err != nil { + return MergedFingerprint{}, false + } + return fp, true +} diff --git a/internal/core/merged_pak_internal_test.go b/internal/core/merged_pak_internal_test.go new file mode 100644 index 0000000..9cbb0c9 --- /dev/null +++ b/internal/core/merged_pak_internal_test.go @@ -0,0 +1,108 @@ +package core + +import "testing" + +func TestMergedFingerprint_Deterministic(t *testing.T) { + f := MergedFingerprint{ + BaseIndexHash: "abc123", + Mods: []MergedFingerprintEntry{ + {SourceID: "icarus", ModID: "bear-mount", Version: "1.0", Checksum: "deadbeef"}, + {SourceID: "icarus", ModID: "wolf-mount", Version: "2.0", Checksum: "cafef00d"}, + }, + } + b1, err := marshalMergedFingerprint(f) + if err != nil { + t.Fatalf("marshal 1: %v", err) + } + b2, err := marshalMergedFingerprint(f) + if err != nil { + t.Fatalf("marshal 2: %v", err) + } + if string(b1) != string(b2) { + t.Errorf("marshal not deterministic: %q vs %q", b1, b2) + } +} + +func TestMergedFingerprintsEqual_IdenticalInputs(t *testing.T) { + f := MergedFingerprint{ + BaseIndexHash: "abc123", + Mods: []MergedFingerprintEntry{{SourceID: "icarus", ModID: "bear-mount", Version: "1.0", Checksum: "deadbeef"}}, + } + eq, err := mergedFingerprintsEqual(f, f) + if err != nil { + t.Fatalf("mergedFingerprintsEqual: %v", err) + } + if !eq { + t.Errorf("identical fingerprints compared unequal") + } +} + +func TestMergedFingerprintsEqual_BaseHashChanged(t *testing.T) { + a := MergedFingerprint{BaseIndexHash: "abc123", Mods: []MergedFingerprintEntry{{SourceID: "icarus", ModID: "m1", Version: "1.0", Checksum: "x"}}} + b := a + b.BaseIndexHash = "def456" + eq, err := mergedFingerprintsEqual(a, b) + if err != nil { + t.Fatalf("mergedFingerprintsEqual: %v", err) + } + if eq { + t.Errorf("base pak change (regeneration trigger) must compare unequal") + } +} + +func TestMergedFingerprintsEqual_ModSetChanged(t *testing.T) { + a := MergedFingerprint{BaseIndexHash: "abc", Mods: []MergedFingerprintEntry{{SourceID: "icarus", ModID: "m1", Version: "1.0", Checksum: "x"}}} + b := MergedFingerprint{BaseIndexHash: "abc", Mods: []MergedFingerprintEntry{ + {SourceID: "icarus", ModID: "m1", Version: "1.0", Checksum: "x"}, + {SourceID: "icarus", ModID: "m2", Version: "1.0", Checksum: "y"}, + }} + eq, err := mergedFingerprintsEqual(a, b) + if err != nil { + t.Fatalf("mergedFingerprintsEqual: %v", err) + } + if eq { + t.Errorf("enabling a mod (regeneration trigger) must compare unequal") + } +} + +func TestMergedFingerprintsEqual_LoadOrderChanged(t *testing.T) { + a := MergedFingerprint{BaseIndexHash: "abc", Mods: []MergedFingerprintEntry{ + {SourceID: "icarus", ModID: "m1", Version: "1.0", Checksum: "x"}, + {SourceID: "icarus", ModID: "m2", Version: "1.0", Checksum: "y"}, + }} + b := MergedFingerprint{BaseIndexHash: "abc", Mods: []MergedFingerprintEntry{ + {SourceID: "icarus", ModID: "m2", Version: "1.0", Checksum: "y"}, + {SourceID: "icarus", ModID: "m1", Version: "1.0", Checksum: "x"}, + }} + eq, err := mergedFingerprintsEqual(a, b) + if err != nil { + t.Fatalf("mergedFingerprintsEqual: %v", err) + } + if eq { + t.Errorf("a load-order swap (regeneration trigger) must compare unequal, got equal") + } +} + +func TestMergedFingerprintsEqual_VersionChanged(t *testing.T) { + a := MergedFingerprint{BaseIndexHash: "abc", Mods: []MergedFingerprintEntry{{SourceID: "icarus", ModID: "m1", Version: "1.0", Checksum: "x"}}} + b := MergedFingerprint{BaseIndexHash: "abc", Mods: []MergedFingerprintEntry{{SourceID: "icarus", ModID: "m1", Version: "2.0", Checksum: "x2"}}} + eq, err := mergedFingerprintsEqual(a, b) + if err != nil { + t.Fatalf("mergedFingerprintsEqual: %v", err) + } + if eq { + t.Errorf("a mod version bump (regeneration trigger) must compare unequal") + } +} + +func TestMergedFingerprintsEqual_EmptyModsBothSides(t *testing.T) { + a := MergedFingerprint{BaseIndexHash: "abc", Mods: nil} + b := MergedFingerprint{BaseIndexHash: "abc", Mods: []MergedFingerprintEntry{}} + eq, err := mergedFingerprintsEqual(a, b) + if err != nil { + t.Fatalf("mergedFingerprintsEqual: %v", err) + } + if !eq { + t.Errorf("nil vs empty Mods slice must still compare equal (both marshal to the same JSON array shape)") + } +} diff --git a/internal/core/merged_pak_test.go b/internal/core/merged_pak_test.go new file mode 100644 index 0000000..f7e80fb --- /dev/null +++ b/internal/core/merged_pak_test.go @@ -0,0 +1,252 @@ +package core_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/core" + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" + "github.com/stretchr/testify/require" +) + +// TestEnabledExmodzSources_OrderMatchesProfileLoadOrderAndSkipsDisabled +// proves enabledExmodzSources returns retained exmodz files in PROFILE +// LOAD ORDER (merge-application order), skips disabled mods entirely, and +// skips a mod's fileIDs that have no retained source (a plain .pak). +func TestEnabledExmodzSources_OrderMatchesProfileLoadOrderAndSkipsDisabled(t *testing.T) { + svc := newFlowsTestService(t) + game := &domain.Game{ID: "icarus", ModPath: t.TempDir(), DeployMode: domain.DeployCompile} + require.NoError(t, svc.AddGame(game)) + + gameCache := svc.GetGameCache(game) + + seedMod := func(sourceID, modID, version string, fileIDs []string, enabled bool) { + for _, fileID := range fileIDs { + require.NoError(t, gameCache.Store(game.ID, sourceID, modID, version, cache.RetainedSourceName(fileID), []byte("exmodz-"+modID+"-"+fileID))) + } + require.NoError(t, svc.SaveInstalledMod(&domain.InstalledMod{ + Mod: domain.Mod{ID: modID, SourceID: sourceID, Name: modID, Version: version, GameID: game.ID}, + ProfileName: "default", + Enabled: enabled, + FileIDs: fileIDs, + UpdatePolicy: domain.UpdateNotify, + })) + } + + // mixedMod has one exmodz fileID and one plain-pak fileID (no retained + // source for the latter) - only the exmodz one should be included. + seedMod("icarus", "second-mod", "1.0", []string{"exmodz-file", "pak-file"}, true) + seedMod("icarus", "first-mod", "1.0", []string{"exmodz-file"}, true) + seedMod("icarus", "disabled-mod", "1.0", []string{"exmodz-file"}, false) + + pm := svc.NewProfileManager() + _, err := pm.Create(game.ID, "default") + require.NoError(t, err) + // Profile load order: first-mod, then second-mod (disabled-mod + // intentionally omitted - membership in Profile.Mods, not just an + // Enabled DB row, is what GetInstalledModsInProfileOrder requires). + require.NoError(t, pm.UpsertMod(game.ID, "default", domain.ModReference{SourceID: "icarus", ModID: "first-mod", Version: "1.0", FileIDs: []string{"exmodz-file"}})) + require.NoError(t, pm.UpsertMod(game.ID, "default", domain.ModReference{SourceID: "icarus", ModID: "second-mod", Version: "1.0", FileIDs: []string{"exmodz-file", "pak-file"}})) + require.NoError(t, pm.UpsertMod(game.ID, "default", domain.ModReference{SourceID: "icarus", ModID: "disabled-mod", Version: "1.0", FileIDs: []string{"exmodz-file"}})) + + sources, err := svc.EnabledExmodzSourcesForTest(game, "default") + require.NoError(t, err) + require.Len(t, sources, 2, "disabled-mod excluded; second-mod's plain-pak fileID excluded") + require.Equal(t, "icarus:first-mod", sources[0].ModRef) + require.Equal(t, "icarus:second-mod", sources[1].ModRef) + + data, err := os.ReadFile(sources[0].ExmodzPath) + require.NoError(t, err) + require.Equal(t, "exmodz-first-mod-exmodz-file", string(data)) +} + +// newMergedPakTestGame builds a DeployCompile game with a registered merge +// compiler and an installed base pak - shared setup for syncMergedPak +// tests. Returns the service, game, and the base pak's own path (so a test +// can rewrite it to simulate a base-pak refresh). +func newMergedPakTestGame(t *testing.T) (*core.Service, *domain.Game, string) { + t.Helper() + + installDir := t.TempDir() + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + writeFakeBasePak(t, basePak) + + svc := newFlowsTestService(t) + src := &fakeCompilerSource{} + svc.RegisterSource(src) + + game := &domain.Game{ + ID: "icarus", InstallPath: installDir, ModPath: t.TempDir(), + DeployMode: domain.DeployCompile, LinkMethod: domain.LinkCopy, + SourceIDs: map[string]string{"fake-compiler": "external-icarus-id"}, + } + require.NoError(t, svc.AddGame(game)) + + pm := svc.NewProfileManager() + _, err := pm.Create(game.ID, "default") + require.NoError(t, err) + + return svc, game, basePak +} + +// seedEnabledExmodzMod installs an ENABLED mod with a retained exmodz file, +// via svc.SaveInstalledMod + profile UpsertMod (matching the real ingest +// shape Task 2/3 produce - a cache entry with a retained source and no +// deployment members). +func seedEnabledExmodzMod(t *testing.T, svc *core.Service, game *domain.Game, sourceID, modID, version, fileID string, exmodzContent []byte) { + t.Helper() + gameCache := svc.GetGameCache(game) + require.NoError(t, gameCache.Store(game.ID, sourceID, modID, version, cache.RetainedSourceName(fileID), exmodzContent)) + require.NoError(t, svc.SaveInstalledMod(&domain.InstalledMod{ + Mod: domain.Mod{ID: modID, SourceID: sourceID, Name: modID, Version: version, GameID: game.ID}, + ProfileName: "default", + Enabled: true, + FileIDs: []string{fileID}, + UpdatePolicy: domain.UpdateNotify, + })) + pm := svc.NewProfileManager() + require.NoError(t, pm.UpsertMod(game.ID, "default", domain.ModReference{SourceID: sourceID, ModID: modID, Version: version, FileIDs: []string{fileID}})) +} + +// TestSyncMergedPak_GeneratesAndDeploys is the happy path: one enabled +// exmodz mod, no merged pak yet - syncMergedPak must generate one and +// deploy it into the game directory. +func TestSyncMergedPak_GeneratesAndDeploys(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-exmodz-bytes")) + + warnings, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + require.Empty(t, warnings) + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + data, err := os.ReadFile(deployedPath) + require.NoError(t, err) + require.Equal(t, "bear-exmodz-bytes", string(data), "fakeCompilerSource's MergeCompile concatenates source bytes - see its own definition") +} + +// TestSyncMergedPak_NoOpWhenUnchanged proves the fingerprint gate actually +// gates: calling syncMergedPak twice with nothing changed must not +// recompile (fakeCompilerSource.compileCalls stays at 1). +func TestSyncMergedPak_NoOpWhenUnchanged(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-exmodz-bytes")) + + _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + + srcRaw, err := svc.GetSource("fake-compiler") + require.NoError(t, err) + src, ok := srcRaw.(*fakeCompilerSource) + require.True(t, ok) + require.Equal(t, 1, src.compileCalls) + + _, err = svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + require.Equal(t, 1, src.compileCalls, "an unchanged fingerprint must not trigger a second merge") +} + +// TestSyncMergedPak_RegeneratesOnModEnable proves enabling a SECOND mod +// (the mod-set changing) triggers regeneration. +func TestSyncMergedPak_RegeneratesOnModEnable(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + + _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "wolf-mount", "1.0", "exmodz-file", []byte("wolf-bytes")) + + warnings, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + require.Empty(t, warnings) + + srcRaw, err := svc.GetSource("fake-compiler") + require.NoError(t, err) + src, ok := srcRaw.(*fakeCompilerSource) + require.True(t, ok) + require.Equal(t, 2, src.compileCalls, "a mod-set change must trigger a second merge") + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + data, err := os.ReadFile(deployedPath) + require.NoError(t, err) + require.Equal(t, "bear-byteswolf-bytes", string(data), "the merged pak must now reflect BOTH mods") +} + +// TestSyncMergedPak_ZeroEnabledMods_UninstallsExistingPak proves the +// uninstall-to-zero case: disabling the LAST enabled exmodz mod must +// remove any previously-deployed merged pak from the game directory. +func TestSyncMergedPak_ZeroEnabledMods_UninstallsExistingPak(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + + _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + _, err = os.Stat(deployedPath) + require.NoError(t, err, "precondition: the merged pak must exist before disabling") + + require.NoError(t, svc.SetModEnabled("fake-compiler", "bear-mount", game.ID, "default", false)) + + _, err = svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + + _, err = os.Stat(deployedPath) + require.True(t, os.IsNotExist(err), "disabling the last exmodz mod must remove the deployed merged pak") +} + +// TestSyncMergedPak_RegeneratesOnBaseHashChange proves a base-pak refresh +// (the "Friday problem", generalized from #196 to the merged model) still +// triggers regeneration. +func TestSyncMergedPak_RegeneratesOnBaseHashChange(t *testing.T) { + svc, game, basePak := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + + _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + + // Rewrite the base pak with different content - a new IndexHash. + writeFakeBasePakWithTable(t, basePak, map[string][]byte{"AI/D_Other.json": []byte(`{"Rows":[{"Name":"x","V":1}]}`)}) + + _, err = svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + + srcRaw, err := svc.GetSource("fake-compiler") + require.NoError(t, err) + src, ok := srcRaw.(*fakeCompilerSource) + require.True(t, ok) + require.Equal(t, 2, src.compileCalls, "a base pak change must trigger a second merge") +} + +// TestSyncMergedPak_NonCompileGame_NoOp: a DeployExtract/DeployCopy game has +// no merged-pak concept at all - syncMergedPak must no-op unconditionally +// (cheap enough to call from every mutation flow regardless of game type). +func TestSyncMergedPak_NonCompileGame_NoOp(t *testing.T) { + svc := newFlowsTestService(t) + game := &domain.Game{ID: "skyrim-se", ModPath: t.TempDir(), DeployMode: domain.DeployExtract} + require.NoError(t, svc.AddGame(game)) + + warnings, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + require.Empty(t, warnings) +} + +// TestSyncMergedPak_AssetCollisionWarningSurfaces proves MergeCompile's own +// warnings (Task 1) propagate all the way out of syncMergedPak. +func TestSyncMergedPak_AssetCollisionWarningSurfaces(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + srcRaw, err := svc.GetSource("fake-compiler") + require.NoError(t, err) + src, ok := srcRaw.(*fakeCompilerSource) + require.True(t, ok) + src.mergeWarnings = []string{"asset collision: fixture warning"} + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + + warnings, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + require.Equal(t, []string{"asset collision: fixture warning"}, warnings) +} diff --git a/internal/core/service_icarus_compile_test.go b/internal/core/service_icarus_compile_test.go index 7856254..735b9d4 100644 --- a/internal/core/service_icarus_compile_test.go +++ b/internal/core/service_icarus_compile_test.go @@ -39,6 +39,7 @@ type fakeCompilerSource struct { downloadURL string compileCalls int validateCalls int + mergeWarnings []string } func (s *fakeCompilerSource) ID() string { return "fake-compiler" } @@ -90,7 +91,7 @@ func (s *fakeCompilerSource) MergeCompile(ctx context.Context, basePakPath strin } out = append(out, data...) } - return nil, os.WriteFile(outputPath, out, 0o644) + return s.mergeWarnings, os.WriteFile(outputPath, out, 0o644) } var ( @@ -98,6 +99,19 @@ var ( _ source.MergeCompiler = (*fakeCompilerSource)(nil) ) +// writeFakeBasePakWithTable is writeFakeBasePak's table-content-controlling +// variant - needed to simulate a base pak refresh (a new IndexHash) for +// staleness tests. +func writeFakeBasePakWithTable(t *testing.T, path string, tables map[string][]byte) { + t.Helper() + w, err := unrealpak.Create(path) + require.NoError(t, err) + for mountPath, data := range tables { + require.NoError(t, w.AddFile(mountPath, data)) + } + require.NoError(t, w.Close()) +} + // failingValidateCompilerSource wraps fakeCompilerSource and always fails // ValidateSource - simulates a corrupt/malformed downloaded .exmodz. type failingValidateCompilerSource struct { diff --git a/internal/domain/mod.go b/internal/domain/mod.go index 430a230..33b8ec9 100644 --- a/internal/domain/mod.go +++ b/internal/domain/mod.go @@ -19,6 +19,12 @@ var UpdateProgressContextKey = &updateProgressKey{} // SourceLocal is the source ID for mods imported from local files const SourceLocal = "local" +// SourceMerged is the source ID for the synthetic, profile-scoped "mod" +// that tracks a game's merged compiled pak (#197 - Icarus's cross-mod +// table merge). Follows the SourceLocal precedent: a reserved sentinel +// string, not a real ModSource registration. +const SourceMerged = "lmm-merged" + // UpdatePolicy determines how a mod handles updates type UpdatePolicy int diff --git a/internal/storage/cache/cache.go b/internal/storage/cache/cache.go index 0307d9a..2bd7019 100644 --- a/internal/storage/cache/cache.go +++ b/internal/storage/cache/cache.go @@ -328,6 +328,22 @@ func RetainedSourceName(fileID string) string { return retainedSourcePrefix + fileID } +// mergeFingerprintMarkerName names the single JSON fingerprint marker a +// merged-pak cache entry carries (#197): what base pak and which +// (source, mod, version, exmodz-checksum) tuples, in order, the pak was +// last built from - so a later staleness check can compare without +// re-deriving the merge. Reserved (ReservedPrefix) so ListFiles/Size/deploy +// skip it like every other lmm bookkeeping entry. +const mergeFingerprintMarkerName = ReservedPrefix + "merge-fingerprint" + +// MergeFingerprintPath returns the reserved on-disk path for versionDir's +// merge-fingerprint marker. Pure naming, like RetainedSourceName - callers +// (internal/core, which owns the MergedFingerprint type and its JSON +// encoding) read/write the actual bytes with ordinary file I/O. +func MergeFingerprintPath(versionDir string) string { + return filepath.Join(versionDir, mergeFingerprintMarkerName) +} + // Store saves a file to the cache func (c *Cache) Store(gameID, sourceID, modID, version, relativePath string, content []byte) error { modPath := c.ModPath(gameID, sourceID, modID, version) diff --git a/internal/storage/cache/cache_test.go b/internal/storage/cache/cache_test.go index 222fa13..ddb1800 100644 --- a/internal/storage/cache/cache_test.go +++ b/internal/storage/cache/cache_test.go @@ -558,3 +558,27 @@ func TestCache_RetainedSourceName_IsReservedAndExcludedFromContent(t *testing.T) func TestCache_RetainedSourceName_UniquePerFileID(t *testing.T) { assert.NotEqual(t, cache.RetainedSourceName("file-a"), cache.RetainedSourceName("file-b")) } + +// TestCache_MergeFingerprintPath_IsReserved pins that the merged pak's +// fingerprint marker (#197) lives under the reserved namespace, like every +// other lmm bookkeeping file. +func TestCache_MergeFingerprintPath_IsReserved(t *testing.T) { + path := cache.MergeFingerprintPath("/some/version/dir") + if !strings.HasPrefix(filepath.Base(path), cache.ReservedPrefix) { + t.Errorf("MergeFingerprintPath = %q, want a reserved-prefixed basename", path) + } +} + +// TestCache_MergeFingerprintPath_ExcludedFromContent proves the fingerprint +// marker is never listed as deployable content, matching every other +// reserved marker's ListFiles exclusion. +func TestCache_MergeFingerprintPath_ExcludedFromContent(t *testing.T) { + c := cache.New(t.TempDir()) + require.NoError(t, c.Store("g", "lmm-merged", "merged-pak", "merged", "zzz_LMM_Merged_P.pak", []byte("pak-bytes"))) + versionDir := c.ModPath("g", "lmm-merged", "merged-pak", "merged") + require.NoError(t, os.WriteFile(cache.MergeFingerprintPath(versionDir), []byte(`{"BaseIndexHash":"abc"}`), 0o644)) + + files, err := c.ListFiles("g", "lmm-merged", "merged-pak", "merged") + require.NoError(t, err) + assert.Equal(t, []string{"zzz_LMM_Merged_P.pak"}, files, "the fingerprint marker must never be listed as deployable content") +} From 79f92bbffe3ab2e8f58ce8a26d6097e063f9feec Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 20:56:11 -0400 Subject: [PATCH 69/96] feat: sync merged pak on enable/disable/uninstall/deploy/switch/update/install/reorder (#197) --- cmd/lmm/profile.go | 2 +- internal/core/flows.go | 85 ++++++ internal/core/merged_pak_hooks_test.go | 132 +++++++++ .../service_apply_recompile_review_test.go | 165 ----------- internal/core/service_apply_recompile_test.go | 265 ------------------ internal/core/service_base_staleness_test.go | 180 ------------ internal/tui/service_core.go | 2 +- 7 files changed, 219 insertions(+), 612 deletions(-) create mode 100644 internal/core/merged_pak_hooks_test.go delete mode 100644 internal/core/service_apply_recompile_review_test.go delete mode 100644 internal/core/service_apply_recompile_test.go delete mode 100644 internal/core/service_base_staleness_test.go diff --git a/cmd/lmm/profile.go b/cmd/lmm/profile.go index f1d6048..15b390c 100644 --- a/cmd/lmm/profile.go +++ b/cmd/lmm/profile.go @@ -828,7 +828,7 @@ func doProfileReorder(service *core.Service, game *domain.Game, args []string) e } } - if err := pm.ReorderMods(game.ID, profileName, newRefs); err != nil { + if err := service.ReorderProfileMods(game.ID, profileName, newRefs); err != nil { return fmt.Errorf("reordering: %w", err) } fmt.Printf("✓ Load order updated for profile %s.\n", profileName) diff --git a/internal/core/flows.go b/internal/core/flows.go index a6bbd9e..efe20db 100644 --- a/internal/core/flows.go +++ b/internal/core/flows.go @@ -16,6 +16,35 @@ import ( "github.com/DonovanMods/linux-mod-manager/internal/storage/config" ) +// ReorderProfileMods persists mods as gameID/profileName's new load order +// (via ProfileManager.ReorderMods) and syncs the merged pak (#197: a +// load-order change is a documented regeneration trigger, since profile +// load order IS merge-application order - see enabledExmodzSources). The +// single seam cmd/lmm and internal/tui both call, replacing their +// previous direct pm.ReorderMods(...) calls (CLI+TUI parity). +// +// A sync failure is non-fatal and returned as part of the SAME error only +// if the reorder itself also failed; a reorder that succeeded but whose +// merged-pak sync failed still returns nil - the reorder took effect, and +// `lmm update`/`lmm verify` are the safety net for a merged pak that +// didn't catch up. Callers wanting to surface a sync warning distinctly +// can call Service.syncMergedPak's own exported test seam directly in a +// follow-up if this proves too quiet in practice; kept simple here to +// match ReorderMods' own existing bare-error signature rather than +// inventing a new result type for one warning slice. +func (s *Service) ReorderProfileMods(gameID, profileName string, mods []domain.ModReference) error { + pm := NewProfileManager(s.configDir, s.db) + if err := pm.ReorderMods(gameID, profileName, mods); err != nil { + return err + } + game, ok := s.games[gameID] + if !ok { + return nil // an unknown game has no merged pak to sync either + } + _, _ = s.syncMergedPak(context.Background(), game, profileName) //nolint:errcheck // best-effort, see doc comment + return nil +} + // EnableResult reports the outcome of EnableMod. Changed is true iff the // mod was actually deployed and flipped to enabled — false (not an error) // when it was already enabled, mirroring EnableMod's pre-Task-6 (bool, @@ -89,6 +118,14 @@ func (s *Service) EnableMod(ctx context.Context, game *domain.Game, profileName, return result, fmt.Errorf("failed to update mod status: %w", err) } + if syncWarnings, syncErr := s.syncMergedPak(ctx, game, profileName); syncErr != nil { + result.Notes = append(result.Notes, fmt.Sprintf("Warning: could not sync merged pak: %v", syncErr)) + } else { + for _, w := range syncWarnings { + result.Notes = append(result.Notes, "Warning: "+w) + } + } + result.Changed = true return result, nil } @@ -162,6 +199,14 @@ func (s *Service) DisableMod(ctx context.Context, game *domain.Game, profileName return result, fmt.Errorf("failed to update mod status: %w", err) } + if syncWarnings, syncErr := s.syncMergedPak(ctx, game, profileName); syncErr != nil { + result.Notes = append(result.Notes, fmt.Sprintf("Warning: could not sync merged pak: %v", syncErr)) + } else { + for _, w := range syncWarnings { + result.Notes = append(result.Notes, "Warning: "+w) + } + } + result.Changed = true return result, nil } @@ -291,6 +336,14 @@ func (s *Service) UninstallMod(ctx context.Context, game *domain.Game, profileNa result.Warnings = append(result.Warnings, fmt.Sprintf("uninstall.after_all hook failed: %v", err)) } + if syncWarnings, syncErr := s.syncMergedPak(ctx, game, profileName); syncErr != nil { + result.Notes = append(result.Notes, fmt.Sprintf("Warning: could not sync merged pak: %v", syncErr)) + } else { + for _, w := range syncWarnings { + result.Notes = append(result.Notes, "Warning: "+w) + } + } + return result, nil } @@ -1815,6 +1868,17 @@ func (s *Service) DeployProfile(ctx context.Context, game *domain.Game, profileN } } + if syncWarnings, syncErr := s.syncMergedPak(ctx, game, profileName); syncErr != nil { + msg := fmt.Sprintf("syncing merged pak: %v", syncErr) + result.Warnings = append(result.Warnings, msg) + emit(DeployProgress{Phase: DeployWarning, Detail: msg}) + } else { + for _, w := range syncWarnings { + result.Warnings = append(result.Warnings, w) + emit(DeployProgress{Phase: DeployWarning, Detail: w}) + } + } + for _, w := range deferredWarnings { emit(w) } @@ -2641,6 +2705,14 @@ func (s *Service) ApplyProfileSwitch(ctx context.Context, game *domain.Game, pla return result, fmt.Errorf("setting default profile: %w", err) } + if syncWarnings, syncErr := s.syncMergedPak(ctx, game, plan.To); syncErr != nil { + result.Notes = append(result.Notes, fmt.Sprintf("Warning: could not sync merged pak: %v", syncErr)) + } else { + for _, w := range syncWarnings { + result.Notes = append(result.Notes, "Warning: "+w) + } + } + return result, nil } @@ -3647,6 +3719,12 @@ func (s *Service) ApplyInstall(ctx context.Context, game *domain.Game, plan *Ins emit(w) } + if syncWarnings, syncErr := s.syncMergedPak(ctx, game, plan.Profile); syncErr != nil { + result.Warnings = append(result.Warnings, fmt.Sprintf("syncing merged pak: %v", syncErr)) + } else { + result.Warnings = append(result.Warnings, syncWarnings...) + } + return result, nil } @@ -4465,6 +4543,13 @@ func (s *Service) ApplyUpdate(ctx context.Context, game *domain.Game, profileNam } result.Applied = append(result.Applied, fmt.Sprintf("%s %s → %s", mod.Name, mod.Version, effectiveVersion)) + + if syncWarnings, syncErr := s.syncMergedPak(ctx, game, profileName); syncErr != nil { + result.Warnings = append(result.Warnings, fmt.Sprintf("syncing merged pak: %v", syncErr)) + } else { + result.Warnings = append(result.Warnings, syncWarnings...) + } + return result, nil } diff --git a/internal/core/merged_pak_hooks_test.go b/internal/core/merged_pak_hooks_test.go new file mode 100644 index 0000000..e67ab60 --- /dev/null +++ b/internal/core/merged_pak_hooks_test.go @@ -0,0 +1,132 @@ +package core_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/core" + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/stretchr/testify/require" +) + +// TestEnableMod_SyncsMergedPak proves enabling an exmodz mod deploys the +// merged pak without a separate `lmm update` step. +func TestEnableMod_SyncsMergedPak(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + require.NoError(t, svc.SetModEnabled("fake-compiler", "bear-mount", game.ID, "default", false)) + + _, err := svc.EnableMod(context.Background(), game, "default", "fake-compiler", "bear-mount") + require.NoError(t, err) + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + _, err = os.Stat(deployedPath) + require.NoError(t, err, "EnableMod must sync the merged pak, not just this mod's own (empty) cache entry") +} + +// TestDisableMod_SyncsMergedPak_RemovesWhenLastModDisabled proves disabling +// the LAST enabled exmodz mod removes the merged pak. +func TestDisableMod_SyncsMergedPak_RemovesWhenLastModDisabled(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + _, err = os.Stat(deployedPath) + require.NoError(t, err) + + _, err = svc.DisableMod(context.Background(), game, "default", "fake-compiler", "bear-mount") + require.NoError(t, err) + + _, err = os.Stat(deployedPath) + require.True(t, os.IsNotExist(err), "DisableMod must sync the merged pak, removing it once the last exmodz mod is disabled") +} + +// TestUninstallMod_SyncsMergedPak_RemovesWhenLastModUninstalled mirrors +// the disable case for a full uninstall. +func TestUninstallMod_SyncsMergedPak_RemovesWhenLastModUninstalled(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + + _, err = svc.UninstallMod(context.Background(), game, "default", "fake-compiler", "bear-mount", core.UninstallOptions{}) + require.NoError(t, err) + + _, err = os.Stat(deployedPath) + require.True(t, os.IsNotExist(err), "UninstallMod must sync the merged pak") +} + +// TestDeployProfile_SyncsMergedPak proves a full `lmm deploy` also +// generates the merged pak (the pre-existing per-mod loop deploys zero +// files for an exmodz mod's own cache entry - Tasks 2/3 - so without this +// hook a fresh deploy would silently produce no merged pak at all). +func TestDeployProfile_SyncsMergedPak(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + + _, err := svc.DeployProfile(context.Background(), game, "default", core.DeployOptions{}, nil) + require.NoError(t, err) + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + data, err := os.ReadFile(deployedPath) + require.NoError(t, err) + require.Equal(t, "bear-bytes", string(data)) +} + +// TestApplyProfileSwitch_SyncsMergedPakForToProfile proves switching TO a +// profile with enabled exmodz mods deploys ITS merged pak (plan.To, not +// plan.From). +func TestApplyProfileSwitch_SyncsMergedPakForToProfile(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + pm := svc.NewProfileManager() + _, err := pm.Create(game.ID, "other") + require.NoError(t, err) + + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + // Move the mod's profile membership to "other" too, so switching there + // has something enabled to merge. + require.NoError(t, pm.UpsertMod(game.ID, "other", domain.ModReference{SourceID: "fake-compiler", ModID: "bear-mount", Version: "1.0", FileIDs: []string{"exmodz-file"}})) + mod, err := svc.GetInstalledMod("fake-compiler", "bear-mount", game.ID, "default") + require.NoError(t, err) + mod.ProfileName = "other" + require.NoError(t, svc.SaveInstalledMod(mod)) + + plan := &core.SwitchPlan{From: "default", To: "other"} + _, err = svc.ApplyProfileSwitch(context.Background(), game, plan, nil) + require.NoError(t, err) + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + _, err = os.Stat(deployedPath) + require.NoError(t, err, "ApplyProfileSwitch must sync the merged pak for the TO profile") +} + +// TestReorderProfileMods_SyncsMergedPak proves a load-order change (a +// documented regeneration trigger) actually reaches the merged pak. +func TestReorderProfileMods_SyncsMergedPak(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-a", []byte("A")) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "wolf-mount", "1.0", "exmodz-b", []byte("B")) + _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + before, err := os.ReadFile(deployedPath) + require.NoError(t, err) + require.Equal(t, "AB", string(before)) + + // Swap load order: wolf-mount now first. + err = svc.ReorderProfileMods(game.ID, "default", []domain.ModReference{ + {SourceID: "fake-compiler", ModID: "wolf-mount", Version: "1.0", FileIDs: []string{"exmodz-b"}}, + {SourceID: "fake-compiler", ModID: "bear-mount", Version: "1.0", FileIDs: []string{"exmodz-a"}}, + }) + require.NoError(t, err) + + _, err = svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + after, err := os.ReadFile(deployedPath) + require.NoError(t, err) + require.Equal(t, "BA", string(after), "reordering must be reflected in a subsequent sync (fingerprint changed)") +} diff --git a/internal/core/service_apply_recompile_review_test.go b/internal/core/service_apply_recompile_review_test.go deleted file mode 100644 index 5ee350c..0000000 --- a/internal/core/service_apply_recompile_review_test.go +++ /dev/null @@ -1,165 +0,0 @@ -package core_test - -import ( - "context" - "errors" - "fmt" - "os" - "path/filepath" - "testing" - - "github.com/DonovanMods/linux-mod-manager/internal/core" - "github.com/DonovanMods/linux-mod-manager/internal/domain" - "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" - "github.com/stretchr/testify/require" -) - -// TestApplyRecompile_RetainedSourceStatError_SurfacesActionably pins the -// #196 review finding: os.Stat's error on the retained source path was -// treated as "missing" unconditionally, masking a genuine permission/I/O -// problem behind misleading "retained source is missing" / redownload- -// fallback text. A non-ENOENT stat error must surface as its own -// actionable failure instead. -// -// A real unreadable-parent fixture would ALSO break ApplyRecompile's own -// prepareStaging seed step (which reads the same version directory before -// this check ever runs), so this exercises the extracted pure classifier -// directly - core.ClassifyRetainedSourceStatError - rather than trying to -// force a filesystem-level permission error through the full call. -func TestClassifyRetainedSourceStatError(t *testing.T) { - t.Run("nil error: present", func(t *testing.T) { - missing, err := core.ClassifyRetainedSourceStatError(nil) - require.False(t, missing) - require.NoError(t, err) - }) - - t.Run("not-exist: missing, no error", func(t *testing.T) { - notExist := &os.PathError{Op: "stat", Path: "/x", Err: os.ErrNotExist} - missing, err := core.ClassifyRetainedSourceStatError(notExist) - require.True(t, missing) - require.NoError(t, err) - }) - - t.Run("permission denied: not missing, actionable error", func(t *testing.T) { - permErr := &os.PathError{Op: "stat", Path: "/x", Err: os.ErrPermission} - missing, err := core.ClassifyRetainedSourceStatError(permErr) - require.False(t, missing, "a permission error must never be folded into 'missing'") - require.Error(t, err) - require.ErrorIs(t, err, permErr) - }) -} - -// TestApplyRecompile_RetainedSourceStatError_Integration proves the -// classifier is actually wired into ApplyRecompile: a stat error that is -// NOT "not exist" must abort with an actionable error, WITHOUT ever -// treating the entry as eligible for the local-mod-fails-loud or -// redownload-fallback text (which would misrepresent a permission/I/O -// problem as "the file is gone"). -// -// Simulated here by replacing the retained source with a directory (a -// real, portable way to make a SECOND os.Stat-adjacent operation fail -// without touching filesystem permissions): os.Stat itself still succeeds -// on a directory, so this proves the narrower regression - that a -// genuinely present (if wrong-shaped) entry is never silently redownloaded -// - while the classifier unit tests above cover the permission-error path -// directly. -func TestApplyRecompile_RetainedSourceIsDirectory_DoesNotSilentlyRedownload(t *testing.T) { - fx := seedCompiledInstalledMod(t, domain.LinkCopy, "fake-compiler", "0000000000000000000000000000000000dead") - - gameCache := fx.svc.GetGameCache(fx.game) - retainedPath := gameCache.GetFilePath(fx.game.ID, "fake-compiler", "bear-mount", "3.3", cache.RetainedSourceName("exmodz-file-id")) - require.NoError(t, os.Remove(retainedPath)) - require.NoError(t, os.Mkdir(retainedPath, 0o755)) - - compiler := &redownloadCompilerSource{ - fakeCompilerSource: &fakeCompilerSource{}, - downloadBody: "should-never-be-fetched", - files: []domain.DownloadableFile{{ID: "exmodz-file-id", FileName: "Bear_Mount.exmodz"}}, - } - compiler.start(t) - fx.svc.RegisterSource(compiler) - - mod, err := fx.svc.GetInstalledMod("fake-compiler", "bear-mount", "icarus", "default") - require.NoError(t, err) - - _, err = fx.svc.ApplyRecompile(context.Background(), fx.game, "default", *mod, nil) - require.Error(t, err, "a present-but-unusable retained source must fail loud, not silently redownload or succeed") -} - -// TestApplyRecompile_RedownloadedFileName_SanitizedAgainstTraversal pins -// the #196 review finding: a source-controlled DownloadableFile.FileName -// joined verbatim into the staging path can traverse outside the intended -// staging directory (e.g. "../evil.exmodz"). The fix must sanitize via -// filepath.Base before joining, mirroring the existing convention used -// elsewhere in this package for exactly this concern (importer.go's -// filepath.Base(archivePath), service.go's filepath.Base(localPath) -// fallback). -func TestApplyRecompile_RedownloadedFileName_SanitizedAgainstTraversal(t *testing.T) { - dataDir := t.TempDir() - installDir := t.TempDir() - basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") - require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) - writeFakeBasePak(t, basePak) - - svc, err := core.NewService(core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: dataDir, CacheDir: t.TempDir()}) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, svc.Close()) }) - - game := &domain.Game{ - ID: "icarus", InstallPath: installDir, ModPath: t.TempDir(), - DeployMode: domain.DeployCompile, LinkMethod: domain.LinkCopy, - SourceIDs: map[string]string{"fake-compiler": "external-icarus-id"}, - } - require.NoError(t, svc.AddGame(game)) - - const modID, version, fileID = "bear-mount", "3.3", "exmodz-file-id" - gameCache := svc.GetGameCache(game) - require.NoError(t, gameCache.Store(game.ID, "fake-compiler", modID, version, "Bear_Mount_P.pak", []byte("stale-compiled-bytes"))) - versionDir := gameCache.ModPath(game.ID, "fake-compiler", modID, version) - require.NoError(t, cache.MarkFileCompleteWithMembers(versionDir, fileID, []string{"Bear_Mount_P.pak"})) - require.NoError(t, cache.MarkBaseIndexHash(versionDir, fileID, "0000000000000000000000000000000000dead")) - // Deliberately NO retained source stored - forces the redownload path, - // which is where the vulnerable filepath.Join(tempDir, match.FileName) - // lives. - - im := &domain.InstalledMod{ - Mod: domain.Mod{ID: modID, SourceID: "fake-compiler", Name: "Bear Mount", Version: version, GameID: game.ID}, - ProfileName: "default", - UpdatePolicy: domain.UpdateNotify, - Enabled: true, - Deployed: true, - LinkMethod: domain.LinkCopy, - FileIDs: []string{fileID}, - } - require.NoError(t, svc.SaveInstalledMod(im)) - installer := svc.GetInstaller(game) - require.NoError(t, installer.Install(context.Background(), game, &im.Mod, "default")) - pm := svc.NewProfileManager() - _, cerr := pm.Create(game.ID, "default") - require.NoError(t, cerr) - require.NoError(t, pm.UpsertMod(game.ID, "default", domain.ModReference{SourceID: "fake-compiler", ModID: modID, Version: version, FileIDs: []string{fileID}})) - - compiler := &redownloadCompilerSource{ - fakeCompilerSource: &fakeCompilerSource{}, - downloadBody: "redownloaded-exmodz-bytes", - files: []domain.DownloadableFile{{ID: fileID, FileName: "../evil-traversal.exmodz"}}, - } - compiler.start(t) - svc.RegisterSource(compiler) - - mod, err := svc.GetInstalledMod("fake-compiler", "bear-mount", "icarus", "default") - require.NoError(t, err) - - _, err = svc.ApplyRecompile(context.Background(), game, "default", *mod, nil) - require.NoError(t, err) - - // newStagingDir("lmm-recompile-*") creates its scratch dir directly - // under dataDir/downloads (Service.stagingRoot) - an UNSANITIZED - // filepath.Join(tempDir, "../evil-traversal.exmodz") climbs exactly one - // level out of that scratch dir, landing at dataDir/downloads/ - // evil-traversal.exmodz. That parent is never removed (only tempDir - // itself is), so an escaped write would persist right here. - escapedPath := filepath.Join(dataDir, "downloads", "evil-traversal.exmodz") - _, statErr := os.Stat(escapedPath) - require.True(t, errors.Is(statErr, os.ErrNotExist), fmt.Sprintf("a traversal filename must never write outside the staging tempDir (found %s)", escapedPath)) -} diff --git a/internal/core/service_apply_recompile_test.go b/internal/core/service_apply_recompile_test.go deleted file mode 100644 index 13bc790..0000000 --- a/internal/core/service_apply_recompile_test.go +++ /dev/null @@ -1,265 +0,0 @@ -package core_test - -import ( - "context" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "testing" - - "github.com/DonovanMods/linux-mod-manager/internal/core" - "github.com/DonovanMods/linux-mod-manager/internal/domain" - "github.com/DonovanMods/linux-mod-manager/internal/source" - "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// redownloadCompilerSource wraps fakeCompilerSource with a real GetModFiles/ -// GetDownloadURL implementation backed by a local HTTP server, so -// ApplyRecompile's "retained source missing -> fall back to re-download" -// leg has something genuine to redownload from. -type redownloadCompilerSource struct { - *fakeCompilerSource - downloadBody string - files []domain.DownloadableFile - srv *httptest.Server -} - -func (s *redownloadCompilerSource) start(t *testing.T) { - t.Helper() - s.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write([]byte(s.downloadBody)) - })) - t.Cleanup(s.srv.Close) -} - -func (s *redownloadCompilerSource) GetModFiles(ctx context.Context, mod *domain.Mod) ([]domain.DownloadableFile, error) { - return s.files, nil -} - -func (s *redownloadCompilerSource) GetDownloadURL(ctx context.Context, mod *domain.Mod, fileID string) (string, error) { - return s.srv.URL, nil -} - -var _ source.ModSource = (*redownloadCompilerSource)(nil) - -// recompileFixture bundles what ApplyRecompile's tests need to assert on: -// the service, game, deployed file's game-dir path, and the base pak path -// (so a test can rewrite it to simulate a base-pak refresh). -type recompileFixture struct { - svc *core.Service - game *domain.Game - deployedPath string // game.ModPath/Bear_Mount_P.pak - basePak string -} - -// seedCompiledInstalledMod builds a DeployCompile game with an installed, -// DEPLOYED compiled mod: cache holds the compiled pak, its retained source, -// and a (possibly stale) base-index marker; the mod is installed via the -// real Installer (so redeploy assertions exercise the real linker) and -// recorded in the DB/profile like any other install. linkMethod is caller- -// controlled because a symlink deployment would trivially reflect the -// atomic cache swap on its own - a copy/hardlink deployment is what proves -// ApplyRecompile's redeploy step actually ran. -func seedCompiledInstalledMod(t *testing.T, linkMethod domain.LinkMethod, sourceID string, recordedHash string) recompileFixture { - t.Helper() - - svc := newFlowsTestService(t) - installDir := t.TempDir() - basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") - require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) - writeFakeBasePak(t, basePak) - - game := &domain.Game{ - ID: "icarus", - InstallPath: installDir, - ModPath: t.TempDir(), - DeployMode: domain.DeployCompile, - LinkMethod: linkMethod, - SourceIDs: map[string]string{"fake-compiler": "external-icarus-id"}, - } - require.NoError(t, svc.AddGame(game)) - - const modID, version, fileID = "bear-mount", "3.3", "exmodz-file-id" - gameCache := svc.GetGameCache(game) - require.NoError(t, gameCache.Store(game.ID, sourceID, modID, version, "Bear_Mount_P.pak", []byte("stale-compiled-bytes"))) - require.NoError(t, gameCache.Store(game.ID, sourceID, modID, version, cache.RetainedSourceName(fileID), []byte("retained-exmodz-bytes"))) - versionDir := gameCache.ModPath(game.ID, sourceID, modID, version) - require.NoError(t, cache.MarkFileCompleteWithMembers(versionDir, fileID, []string{"Bear_Mount_P.pak"})) - if recordedHash != "" { - require.NoError(t, cache.MarkBaseIndexHash(versionDir, fileID, recordedHash)) - } - - im := &domain.InstalledMod{ - Mod: domain.Mod{ID: modID, SourceID: sourceID, Name: "Bear Mount", Version: version, GameID: game.ID}, - ProfileName: "default", - UpdatePolicy: domain.UpdateNotify, - Enabled: true, - Deployed: true, - LinkMethod: linkMethod, - FileIDs: []string{fileID}, - } - require.NoError(t, svc.SaveInstalledMod(im)) - - installer := svc.GetInstaller(game) - require.NoError(t, installer.Install(context.Background(), game, &im.Mod, "default")) - - pm := svc.NewProfileManager() - _, cerr := pm.Create(game.ID, "default") - require.NoError(t, cerr) - require.NoError(t, pm.UpsertMod(game.ID, "default", domain.ModReference{SourceID: sourceID, ModID: modID, Version: version, FileIDs: []string{fileID}})) - - return recompileFixture{svc: svc, game: game, deployedPath: filepath.Join(game.ModPath, "Bear_Mount_P.pak"), basePak: basePak} -} - -// TestApplyRecompile_OfflineFromRetainedSource_Redeploys is the happy path: -// a stale compile recompiles from its retained .exmodz with no network -// access at all (no source registered), lands the fresh bytes in the cache -// under the SAME name, records the live base pak's fingerprint, and -// redeploys - proven with LinkCopy so the on-disk deployed file can only -// carry the new content if ReplaceForUpdate actually ran. -func TestApplyRecompile_OfflineFromRetainedSource_Redeploys(t *testing.T) { - fx := seedCompiledInstalledMod(t, domain.LinkCopy, "fake-compiler", "0000000000000000000000000000000000dead") - liveHash := basePakIndexHash(t, fx.basePak) - - compiler := &fakeCompilerSource{} - fx.svc.RegisterSource(compiler) - - mod, err := fx.svc.GetInstalledMod("fake-compiler", "bear-mount", "icarus", "default") - require.NoError(t, err) - - result, err := fx.svc.ApplyRecompile(context.Background(), fx.game, "default", *mod, nil) - require.NoError(t, err) - require.Equal(t, []string{"Bear_Mount_P.pak"}, result.Applied) - require.Equal(t, 1, compiler.compileCalls) - - // fakeCompilerSource.Compile copies sourceFilePath's bytes through - // unchanged - the retained source's content, proving it (not a - // redownload) was used. - deployedData, err := os.ReadFile(fx.deployedPath) - require.NoError(t, err) - assert.Equal(t, "retained-exmodz-bytes", string(deployedData), "redeploy must reflect the freshly recompiled bytes") - - gameCache := fx.svc.GetGameCache(fx.game) - hashes, err := gameCache.BaseIndexHashes(fx.game.ID, "fake-compiler", "bear-mount", "3.3") - require.NoError(t, err) - assert.Equal(t, liveHash, hashes["exmodz-file-id"], "the recompile must record the CURRENT live base pak hash") -} - -// TestApplyRecompile_LockedRefRefuses mirrors -// TestApplyUpdate_LockedRefRefusesUpdate exactly (#196: lock-wins) - a -// locked mod's files must never be touched by a recompile. -func TestApplyRecompile_LockedRefRefuses(t *testing.T) { - fx := seedCompiledInstalledMod(t, domain.LinkCopy, "fake-compiler", "0000000000000000000000000000000000dead") - fx.svc.RegisterSource(&fakeCompilerSource{}) - - pm := fx.svc.NewProfileManager() - require.NoError(t, pm.SetModLock(fx.game.ID, "default", "fake-compiler", "bear-mount", "")) - - mod, err := fx.svc.GetInstalledMod("fake-compiler", "bear-mount", "icarus", "default") - require.NoError(t, err) - - before, err := os.ReadFile(fx.deployedPath) - require.NoError(t, err) - - _, err = fx.svc.ApplyRecompile(context.Background(), fx.game, "default", *mod, nil) - require.Error(t, err) - assert.ErrorIs(t, err, core.ErrModLocked) - assert.Contains(t, err.Error(), "locked at v") - - after, err := os.ReadFile(fx.deployedPath) - require.NoError(t, err) - assert.Equal(t, before, after, "a locked mod's deployed files must never be touched") -} - -// TestApplyRecompile_PinnedModRecompiles proves pinning does NOT block -// ApplyRecompile (#196 design point 3: pinning fixes the mod VERSION, not -// the base pak) - only ApplyUpdate/UpdateCheckable gate on UpdatePinned. -func TestApplyRecompile_PinnedModRecompiles(t *testing.T) { - fx := seedCompiledInstalledMod(t, domain.LinkCopy, "fake-compiler", "0000000000000000000000000000000000dead") - fx.svc.RegisterSource(&fakeCompilerSource{}) - - mod, err := fx.svc.GetInstalledMod("fake-compiler", "bear-mount", "icarus", "default") - require.NoError(t, err) - mod.UpdatePolicy = domain.UpdatePinned - - _, err = fx.svc.ApplyRecompile(context.Background(), fx.game, "default", *mod, nil) - require.NoError(t, err, "ApplyRecompile itself must not gate on UpdatePolicy") -} - -// TestApplyRecompile_LocalModMissingRetainedSource_FailsLoud: a pure local -// import has no remote to fall back to - a missing retained source must -// fail loud with an actionable remedy, never silently skip or fabricate -// content. -func TestApplyRecompile_LocalModMissingRetainedSource_FailsLoud(t *testing.T) { - fx := seedCompiledInstalledMod(t, domain.LinkCopy, domain.SourceLocal, "0000000000000000000000000000000000dead") - - gameCache := fx.svc.GetGameCache(fx.game) - retainedPath := gameCache.GetFilePath(fx.game.ID, domain.SourceLocal, "bear-mount", "3.3", cache.RetainedSourceName("exmodz-file-id")) - require.NoError(t, os.Remove(retainedPath)) - - fx.svc.RegisterSource(&fakeCompilerSource{}) - - mod, err := fx.svc.GetInstalledMod(domain.SourceLocal, "bear-mount", "icarus", "default") - require.NoError(t, err) - - _, err = fx.svc.ApplyRecompile(context.Background(), fx.game, "default", *mod, nil) - require.Error(t, err) - assert.Contains(t, err.Error(), "retained compile source") - assert.Contains(t, err.Error(), "no remote source") -} - -// TestApplyRecompile_MissingRetainedSource_FallsBackToRedownload proves the -// #196 design's "fallback: re-download" leg for a mod with a REAL source -// connection (a download-compiled entry: fileID is that source's actual -// DownloadableFile.ID, so GetModFiles/GetDownloadURL can resolve it). -func TestApplyRecompile_MissingRetainedSource_FallsBackToRedownload(t *testing.T) { - fx := seedCompiledInstalledMod(t, domain.LinkCopy, "fake-compiler", "0000000000000000000000000000000000dead") - - gameCache := fx.svc.GetGameCache(fx.game) - retainedPath := gameCache.GetFilePath(fx.game.ID, "fake-compiler", "bear-mount", "3.3", cache.RetainedSourceName("exmodz-file-id")) - require.NoError(t, os.Remove(retainedPath)) - - compiler := &redownloadCompilerSource{ - fakeCompilerSource: &fakeCompilerSource{}, - downloadBody: "redownloaded-exmodz-bytes", - files: []domain.DownloadableFile{{ID: "exmodz-file-id", FileName: "Bear_Mount.exmodz"}}, - } - compiler.start(t) - fx.svc.RegisterSource(compiler) - - mod, err := fx.svc.GetInstalledMod("fake-compiler", "bear-mount", "icarus", "default") - require.NoError(t, err) - - result, err := fx.svc.ApplyRecompile(context.Background(), fx.game, "default", *mod, nil) - require.NoError(t, err) - require.Equal(t, []string{"Bear_Mount_P.pak"}, result.Applied) - - deployedData, err := os.ReadFile(fx.deployedPath) - require.NoError(t, err) - assert.Equal(t, "redownloaded-exmodz-bytes", string(deployedData)) -} - -// TestApplyRecompile_NoCompiledEntries_FailsLoud: a mod with no base-index -// markers at all (never compiled) has nothing for ApplyRecompile to do - -// callers should never route such a mod here, but the gate must still fail -// loud rather than silently no-op if one slips through. -func TestApplyRecompile_NoCompiledEntries_FailsLoud(t *testing.T) { - svc := newFlowsTestService(t) - installDir := t.TempDir() - basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") - require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) - writeFakeBasePak(t, basePak) - - game := &domain.Game{ID: "icarus", InstallPath: installDir, ModPath: t.TempDir(), DeployMode: domain.DeployCompile} - require.NoError(t, svc.AddGame(game)) - svc.RegisterSource(&fakeCompilerSource{}) - - mod := domain.InstalledMod{Mod: domain.Mod{ID: "plain-pak-mod", SourceID: "fake-compiler", Name: "Plain Pak", Version: "1.0", GameID: "icarus"}} - - _, err := svc.ApplyRecompile(context.Background(), game, "default", mod, nil) - require.Error(t, err) - assert.Contains(t, err.Error(), "no compiled entries") -} diff --git a/internal/core/service_base_staleness_test.go b/internal/core/service_base_staleness_test.go deleted file mode 100644 index a7670ad..0000000 --- a/internal/core/service_base_staleness_test.go +++ /dev/null @@ -1,180 +0,0 @@ -package core_test - -import ( - "context" - "os" - "path/filepath" - "testing" - - "github.com/DonovanMods/linux-mod-manager/internal/core" - "github.com/DonovanMods/linux-mod-manager/internal/domain" - "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" - "github.com/stretchr/testify/require" -) - -// newStalenessTestService builds a DeployCompile game with a real (fixture) -// base pak installed at installDir, and a service whose cache/config live -// under fresh temp dirs. Returns the service, the game, and the base pak's -// path so tests can rewrite it to simulate a base-pak refresh. -func newStalenessTestService(t *testing.T) (*core.Service, *domain.Game, string) { - t.Helper() - - installDir := t.TempDir() - basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") - require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) - writeFakeBasePak(t, basePak) - - cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} - svc, err := core.NewService(cfg) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, svc.Close()) }) - - game := &domain.Game{ID: "icarus", InstallPath: installDir, ModPath: t.TempDir(), DeployMode: domain.DeployCompile} - require.NoError(t, svc.AddGame(game)) - - return svc, game, basePak -} - -// seedCompiledMod stages a fake compiled entry directly through the cache -// (bypassing Compile/Importer entirely - CheckBaseStaleness only reads -// markers, so this is a faster, more direct way to set up its inputs than -// driving a full compile), recording fingerprint as the file's base-index -// hash if non-empty (empty simulates a pre-#196 entry with NO marker at -// all). -func seedCompiledMod(t *testing.T, svc *core.Service, game *domain.Game, mod domain.InstalledMod, fingerprint string) { - t.Helper() - gameCache := svc.GetGameCache(game) - versionDir := gameCache.ModPath(game.ID, mod.SourceID, mod.ID, mod.Version) - require.NoError(t, os.MkdirAll(versionDir, 0755)) - require.NoError(t, os.WriteFile(filepath.Join(versionDir, "Fake_P.pak"), []byte("compiled"), 0o644)) - if fingerprint != "" { - require.NoError(t, cache.MarkBaseIndexHash(versionDir, "fake-file-id", fingerprint)) - } -} - -func TestCheckBaseStaleness_FingerprintMatch_NotStale(t *testing.T) { - svc, game, basePak := newStalenessTestService(t) - liveHash := basePakIndexHash(t, basePak) - - mod := domain.InstalledMod{Mod: domain.Mod{ID: "bear-mount", SourceID: "fake-compiler", Version: "1.0"}} - seedCompiledMod(t, svc, game, mod, liveHash) - - stale, err := svc.CheckBaseStaleness(game, []domain.InstalledMod{mod}) - require.NoError(t, err) - require.Empty(t, stale, "a fingerprint matching the live base pak must not be reported stale") -} - -func TestCheckBaseStaleness_FingerprintMismatch_Stale(t *testing.T) { - svc, game, _ := newStalenessTestService(t) - - mod := domain.InstalledMod{Mod: domain.Mod{ID: "bear-mount", SourceID: "fake-compiler", Version: "1.0"}} - seedCompiledMod(t, svc, game, mod, "0000000000000000000000000000000000dead") // deliberately wrong - - stale, err := svc.CheckBaseStaleness(game, []domain.InstalledMod{mod}) - require.NoError(t, err) - require.Len(t, stale, 1) - require.True(t, stale[0].RecompileNeeded) - require.Equal(t, mod.Version, stale[0].NewVersion, "NewVersion must equal the current version - the mod hasn't changed, only the base pak has") - require.Equal(t, mod.ID, stale[0].InstalledMod.ID) -} - -// TestCheckBaseStaleness_MissingFingerprint_NotStale pins the #196-review -// amendment: a compiled entry with NO base-index marker (predates #196, or -// is actually a never-compiled prebuilt .pak - the two are locally -// indistinguishable) is skipped, not flagged. Flagging it would false- -// positive forever on plain prebuilt .pak mods, which a DeployCompile -// game's catalog can also legitimately serve (isExmodzFile only routes -// .exmodz through Compile). -func TestCheckBaseStaleness_MissingFingerprint_NotStale(t *testing.T) { - svc, game, _ := newStalenessTestService(t) - - mod := domain.InstalledMod{Mod: domain.Mod{ID: "bear-mount", SourceID: "fake-compiler", Version: "1.0"}} - seedCompiledMod(t, svc, game, mod, "") // no marker at all - - stale, err := svc.CheckBaseStaleness(game, []domain.InstalledMod{mod}) - require.NoError(t, err) - require.Empty(t, stale, "a missing fingerprint must be skipped, not flagged stale") -} - -// TestCheckBaseStaleness_PinnedModIncluded pins design point 3: pinning -// fixes the mod VERSION, not the base pak, so a pinned mod's staleness must -// still be reported (unlike Updater.CheckUpdates, which filters pinned mods -// out entirely via UpdateCheckable). -func TestCheckBaseStaleness_PinnedModIncluded(t *testing.T) { - svc, game, _ := newStalenessTestService(t) - - mod := domain.InstalledMod{Mod: domain.Mod{ID: "bear-mount", SourceID: "fake-compiler", Version: "1.0"}, UpdatePolicy: domain.UpdatePinned} - seedCompiledMod(t, svc, game, mod, "0000000000000000000000000000000000dead") - - stale, err := svc.CheckBaseStaleness(game, []domain.InstalledMod{mod}) - require.NoError(t, err) - require.Len(t, stale, 1, "a pinned mod must still be checked for base staleness") -} - -// TestCheckBaseStaleness_LocalModIncluded: a pure local import (SourceID == -// domain.SourceLocal) has no remote to check, but it CAN go stale against -// the base pak - this check is entirely local/offline, so it must not skip -// local mods the way Updater.CheckUpdates does. -func TestCheckBaseStaleness_LocalModIncluded(t *testing.T) { - svc, game, _ := newStalenessTestService(t) - - mod := domain.InstalledMod{Mod: domain.Mod{ID: "bear-mount", SourceID: domain.SourceLocal, Version: "1.0"}} - seedCompiledMod(t, svc, game, mod, "0000000000000000000000000000000000dead") - - stale, err := svc.CheckBaseStaleness(game, []domain.InstalledMod{mod}) - require.NoError(t, err) - require.Len(t, stale, 1) -} - -// TestCheckBaseStaleness_NonCompileGame_NoOp: a DeployExtract/DeployCopy -// game has no base pak concept at all - CheckBaseStaleness must be an -// unconditional no-op rather than erroring on a missing base pak path. -func TestCheckBaseStaleness_NonCompileGame_NoOp(t *testing.T) { - cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} - svc, err := core.NewService(cfg) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, svc.Close()) }) - - game := &domain.Game{ID: "skyrim-se", ModPath: t.TempDir(), DeployMode: domain.DeployExtract} - require.NoError(t, svc.AddGame(game)) - - mod := domain.InstalledMod{Mod: domain.Mod{ID: "some-mod", SourceID: "nexusmods", Version: "1.0"}} - stale, err := svc.CheckBaseStaleness(game, []domain.InstalledMod{mod}) - require.NoError(t, err) - require.Empty(t, stale) -} - -// TestCheckGameUpdates_MergesStalenessWithoutDuplicating proves the -// combined seam CLI/TUI both use: a mod with a REAL update available is not -// separately duplicated as a staleness row even when it's also stale, and a -// mod with ONLY staleness (no real update) is still surfaced. -func TestCheckGameUpdates_MergesStalenessWithoutDuplicating(t *testing.T) { - svc, game, _ := newStalenessTestService(t) - - src := &updateMockSource{id: "fake-compiler", currentMod: &domain.Mod{ID: "has-real-update", Version: "2.0"}} - svc.RegisterSource(src) - - realUpdateMod := domain.InstalledMod{Mod: domain.Mod{ID: "has-real-update", SourceID: "fake-compiler", Version: "1.0"}} - staleOnlyMod := domain.InstalledMod{Mod: domain.Mod{ID: "stale-only", SourceID: "fake-compiler", Version: "1.0"}} - seedCompiledMod(t, svc, game, realUpdateMod, "0000000000000000000000000000000000dead") - seedCompiledMod(t, svc, game, staleOnlyMod, "0000000000000000000000000000000000dead") - - updates, err := svc.CheckGameUpdates(context.Background(), game, []domain.InstalledMod{realUpdateMod, staleOnlyMod}) - require.NoError(t, err) - require.Len(t, updates, 2, "one real-update row + one staleness-only row, no duplicate for the mod with both") - - byID := map[string]domain.Update{} - for _, u := range updates { - byID[u.InstalledMod.ID] = u - } - - real, ok := byID["has-real-update"] - require.True(t, ok) - require.Equal(t, "2.0", real.NewVersion) - require.False(t, real.RecompileNeeded, "a real version update supersedes the staleness row - recompiling happens as part of applying it") - - stale, ok := byID["stale-only"] - require.True(t, ok) - require.True(t, stale.RecompileNeeded) - require.Equal(t, "1.0", stale.NewVersion) -} diff --git a/internal/tui/service_core.go b/internal/tui/service_core.go index 166a368..420374e 100644 --- a/internal/tui/service_core.go +++ b/internal/tui/service_core.go @@ -975,7 +975,7 @@ func (p *coreProvider) ReorderMods(_ context.Context, orderedKeys []string) (Act }) } - if err := pm.ReorderMods(game.ID, profileName, mods); err != nil { + if err := p.svc.ReorderProfileMods(game.ID, profileName, mods); err != nil { return ActionOutcome{}, fmt.Errorf("reordering profile %s: %w", profileName, err) } return ActionOutcome{Message: "load order updated"}, nil From 4cc3adee72381165290c54d1ce6491ec68d6a018 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 20:56:22 -0400 Subject: [PATCH 70/96] feat: CheckMergedPakStaleness + ApplyMergedPakRegen, retire per-mod #196 staleness (#197) --- internal/core/merged_pak.go | 147 +++++++-- internal/core/merged_pak_staleness_test.go | 95 ++++++ internal/core/merged_pak_test.go | 8 + .../core/service_download_traversal_test.go | 34 +- internal/core/updater.go | 297 +----------------- internal/storage/cache/cache.go | 11 +- 6 files changed, 259 insertions(+), 333 deletions(-) create mode 100644 internal/core/merged_pak_staleness_test.go diff --git a/internal/core/merged_pak.go b/internal/core/merged_pak.go index 7345d88..e102add 100644 --- a/internal/core/merged_pak.go +++ b/internal/core/merged_pak.go @@ -166,9 +166,9 @@ func (s *Service) syncMergedPak(ctx context.Context, game *domain.Game, profileN return nil, nil } - sources, err := s.enabledExmodzSources(game, profileName) + current, sources, err := s.currentMergedFingerprint(game, profileName) if err != nil { - return nil, fmt.Errorf("listing enabled exmodz mods: %w", err) + return nil, err } gameCache := s.GetGameCache(game) @@ -193,38 +193,6 @@ func (s *Service) syncMergedPak(ctx context.Context, game *domain.Game, profileN if err != nil { return nil, err } - liveHash, err := basePakIndexHash(basePakPath) - if err != nil { - return nil, fmt.Errorf("reading base pak for merge fingerprint: %w", err) - } - - current := MergedFingerprint{BaseIndexHash: liveHash} - for _, src := range sources { - sum, herr := md5File(src.ExmodzPath) - if herr != nil { - return nil, fmt.Errorf("hashing %s: %w", src.ExmodzPath, herr) - } - sourceID, modID, _ := strings.Cut(src.ModRef, ":") - current.Mods = append(current.Mods, MergedFingerprintEntry{ - SourceID: sourceID, ModID: modID, Checksum: sum, - }) - } - // Version is not carried on source.MergeSource (it only needs ModRef + - // ExmodzPath for the merge itself) - resolved separately here so - // enabledExmodzSources' own signature stays minimal. Re-fetching the - // installed mods once more is cheap (small profiles) and keeps - // enabledExmodzSources' contract focused on ONE job. - mods, err := s.GetInstalledModsInProfileOrder(game.ID, profileName) - if err != nil { - return nil, fmt.Errorf("loading profile mods: %w", err) - } - versionByRef := make(map[string]string, len(mods)) - for _, m := range mods { - versionByRef[m.SourceID+":"+m.ID] = m.Version - } - for i, src := range sources { - current.Mods[i].Version = versionByRef[src.ModRef] - } cachePath := gameCache.ModPath(game.ID, domain.SourceMerged, mergedPakModID, mergedPakVersion) if stored, ok := readMergedFingerprint(cachePath); ok { @@ -294,3 +262,114 @@ func readMergedFingerprint(cachePath string) (fp MergedFingerprint, ok bool) { } return fp, true } + +// currentMergedFingerprint computes what game+profileName's merged pak +// SHOULD look like right now: the live base pak's IndexHash plus every +// currently-enabled exmodz mod's identity/version/content checksum, in +// profile load order. Returns a nil sources/zero-value fingerprint (not an +// error) when there is nothing to merge - callers distinguish "nothing to +// do" from "failed to compute" via the returned slice's length, exactly +// like syncMergedPak's own zero-sources branch does. +func (s *Service) currentMergedFingerprint(game *domain.Game, profileName string) (MergedFingerprint, []source.MergeSource, error) { + sources, err := s.enabledExmodzSources(game, profileName) + if err != nil { + return MergedFingerprint{}, nil, fmt.Errorf("listing enabled exmodz mods: %w", err) + } + if len(sources) == 0 { + return MergedFingerprint{}, sources, nil + } + + basePakPath, err := resolveBasePak(game) + if err != nil { + return MergedFingerprint{}, sources, err + } + liveHash, err := basePakIndexHash(basePakPath) + if err != nil { + return MergedFingerprint{}, sources, fmt.Errorf("reading base pak for merge fingerprint: %w", err) + } + + current := MergedFingerprint{BaseIndexHash: liveHash} + for _, src := range sources { + sum, herr := md5File(src.ExmodzPath) + if herr != nil { + return MergedFingerprint{}, sources, fmt.Errorf("hashing %s: %w", src.ExmodzPath, herr) + } + sourceID, modID, _ := strings.Cut(src.ModRef, ":") + current.Mods = append(current.Mods, MergedFingerprintEntry{SourceID: sourceID, ModID: modID, Checksum: sum}) + } + + mods, err := s.GetInstalledModsInProfileOrder(game.ID, profileName) + if err != nil { + return MergedFingerprint{}, sources, fmt.Errorf("loading profile mods: %w", err) + } + versionByRef := make(map[string]string, len(mods)) + for _, m := range mods { + versionByRef[m.SourceID+":"+m.ID] = m.Version + } + for i, src := range sources { + current.Mods[i].Version = versionByRef[src.ModRef] + } + + return current, sources, nil +} + +// CheckMergedPakStaleness reports whether game+profileName's merged pak no +// longer matches the current enabled-mod set/order/versions/base pak +// (#197, generalizing #196's per-mod CheckBaseStaleness to the merged +// model). Returns nil, nil - not an error - when the merged pak is +// up to date, when there is nothing to merge (zero enabled exmodz mods), +// or when game is not a DeployCompile game. +func (s *Service) CheckMergedPakStaleness(game *domain.Game, profileName string) (*domain.Update, error) { + if game.DeployMode != domain.DeployCompile { + return nil, nil + } + + current, sources, err := s.currentMergedFingerprint(game, profileName) + if err != nil { + return nil, err + } + if len(sources) == 0 { + return nil, nil + } + + gameCache := s.GetGameCache(game) + cachePath := gameCache.ModPath(game.ID, domain.SourceMerged, mergedPakModID, mergedPakVersion) + stored, ok := readMergedFingerprint(cachePath) + if ok { + if eq, eqErr := mergedFingerprintsEqual(current, stored); eqErr == nil && eq { + return nil, nil + } + } + + return &domain.Update{ + InstalledMod: domain.InstalledMod{ + Mod: domain.Mod{ + ID: mergedPakModID, SourceID: domain.SourceMerged, + Name: "Icarus Merged Pak", Version: mergedPakVersion, GameID: game.ID, + }, + }, + NewVersion: mergedPakVersion, + RecompileNeeded: true, + }, nil +} + +// ApplyMergedPakRegen regenerates game+profileName's merged pak (#197 - +// replaces #196's per-mod ApplyRecompile). No lock gate: a locked mod's +// retained exmodz still participates in every re-merge (design decision 3 +// - locking pins THAT mod's own version, it does not freeze the whole +// merged pak or exclude the mod's diff; reading a locked mod's retained +// source to feed the merge is not "touching" it in the sense a lock +// protects against). +func (s *Service) ApplyMergedPakRegen(ctx context.Context, game *domain.Game, profileName string, progress func(DeployProgress)) (*UpdateApplyResult, error) { + result := &UpdateApplyResult{} + warnings, err := s.syncMergedPak(ctx, game, profileName) + if err != nil { + return result, err + } + result.Warnings = warnings + result.Applied = []string{mergedPakFileName} + if progress != nil { + progress(DeployProgress{Phase: UpdateDownloadDone}) + } + return result, nil +} diff --git a/internal/core/merged_pak_staleness_test.go b/internal/core/merged_pak_staleness_test.go new file mode 100644 index 0000000..6a41e0f --- /dev/null +++ b/internal/core/merged_pak_staleness_test.go @@ -0,0 +1,95 @@ +package core_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/stretchr/testify/require" +) + +func TestCheckMergedPakStaleness_NotStaleWhenUnchanged(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + + upd, err := svc.CheckMergedPakStaleness(game, "default") + require.NoError(t, err) + require.Nil(t, upd, "an up-to-date merged pak must not be reported stale") +} + +func TestCheckMergedPakStaleness_StaleAfterModEnable(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "wolf-mount", "1.0", "exmodz-file", []byte("wolf-bytes")) + + upd, err := svc.CheckMergedPakStaleness(game, "default") + require.NoError(t, err) + require.NotNil(t, upd) + require.True(t, upd.RecompileNeeded) + require.Equal(t, upd.InstalledMod.Version, upd.NewVersion, "a staleness row has no real version change") +} + +func TestCheckMergedPakStaleness_NilWhenNoMergedPakEverGenerated(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + upd, err := svc.CheckMergedPakStaleness(game, "default") + require.NoError(t, err) + require.Nil(t, upd, "zero enabled exmodz mods means nothing to report - not an error, not a staleness row") +} + +func TestCheckMergedPakStaleness_NonCompileGame_Nil(t *testing.T) { + svc := newFlowsTestService(t) + game := &domain.Game{ID: "skyrim-se", ModPath: t.TempDir(), DeployMode: domain.DeployExtract} + require.NoError(t, svc.AddGame(game)) + upd, err := svc.CheckMergedPakStaleness(game, "default") + require.NoError(t, err) + require.Nil(t, upd) +} + +// TestApplyMergedPakRegen_Regenerates proves the apply-side wiring: given +// a stale merged pak, applying regenerates and redeploys it. +func TestApplyMergedPakRegen_Regenerates(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "wolf-mount", "1.0", "exmodz-file", []byte("wolf-bytes")) + + result, err := svc.ApplyMergedPakRegen(context.Background(), game, "default", nil) + require.NoError(t, err) + require.NotNil(t, result) + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + data, err := os.ReadFile(deployedPath) + require.NoError(t, err) + require.Equal(t, "bear-byteswolf-bytes", string(data)) +} + +// TestApplyMergedPakRegen_LockedModDiffStillParticipates is the dedicated +// coordinator-flagged design-decision test - see Task 13 for the FULL +// suite; this is the minimal smoke case proving a LOCKED mod's retained +// exmodz is not excluded from a merge triggered by an UNLOCKED mod's +// change. +func TestApplyMergedPakRegen_LockedModDiffStillParticipates(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("locked-bear-bytes")) + pm := svc.NewProfileManager() + require.NoError(t, pm.SetModLock(game.ID, "default", "fake-compiler", "bear-mount", "")) + + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "wolf-mount", "1.0", "exmodz-file", []byte("wolf-bytes")) + + _, err := svc.ApplyMergedPakRegen(context.Background(), game, "default", nil) + require.NoError(t, err, "a locked mod elsewhere in the profile must not block the merge") + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + data, err := os.ReadFile(deployedPath) + require.NoError(t, err) + require.Contains(t, string(data), "locked-bear-bytes", "the locked mod's diff must still be included in the merge") + require.Contains(t, string(data), "wolf-bytes") +} diff --git a/internal/core/merged_pak_test.go b/internal/core/merged_pak_test.go index f7e80fb..089584d 100644 --- a/internal/core/merged_pak_test.go +++ b/internal/core/merged_pak_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "strings" "testing" "github.com/DonovanMods/linux-mod-manager/internal/core" @@ -25,6 +26,13 @@ func TestEnabledExmodzSources_OrderMatchesProfileLoadOrderAndSkipsDisabled(t *te seedMod := func(sourceID, modID, version string, fileIDs []string, enabled bool) { for _, fileID := range fileIDs { + // Only an "exmodz"-named fileID gets a retained source, mirroring + // the real ingest shape (Task 2/3): a plain .pak fileID is never + // retained, so enabledExmodzSources must skip it via the + // os.Stat check, not just by naming convention. + if !strings.Contains(fileID, "exmodz") { + continue + } require.NoError(t, gameCache.Store(game.ID, sourceID, modID, version, cache.RetainedSourceName(fileID), []byte("exmodz-"+modID+"-"+fileID))) } require.NoError(t, svc.SaveInstalledMod(&domain.InstalledMod{ diff --git a/internal/core/service_download_traversal_test.go b/internal/core/service_download_traversal_test.go index f454d4f..8a7c084 100644 --- a/internal/core/service_download_traversal_test.go +++ b/internal/core/service_download_traversal_test.go @@ -10,6 +10,7 @@ import ( "github.com/DonovanMods/linux-mod-manager/internal/core" "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" "github.com/stretchr/testify/require" ) @@ -62,14 +63,14 @@ func TestDownloadModToCache_TraversalFileName_SanitizedAgainstEscape(t *testing. require.Equal(t, []string{"evil-traversal.zip"}, files, "the sanitized (Base'd) filename is what must actually land in the cache") } -// TestDownloadMod_DeployCompile_TraversalFileName_SanitizedAgainstEscape -// covers the third #196-review site in the same function: the DeployCompile -// branch derives destName via compiledFileName(file.FileName), which only -// trims/adds a suffix - it does not strip directory components, so a -// traversal payload in the STEM (e.g. "../evil.exmodz") survives into -// destName unless separately re-sanitized before the final -// filepath.Join(stagePath, destName). -func TestDownloadMod_DeployCompile_TraversalFileName_SanitizedAgainstEscape(t *testing.T) { +// TestDownloadMod_DeployCompile_TraversalFileID_SanitizedAgainstEscape +// covers the #197-era equivalent of the #196-review site above: the +// DeployCompile branch now retains the .exmodz under +// cache.RetainedSourceName(file.ID) instead of compiling a per-mod pak +// named from file.FileName - file.ID is exactly as source-controlled as +// FileName was, so a traversal payload in the ID (e.g. "../evil-id") must +// not survive into the retained source's on-disk name either. +func TestDownloadMod_DeployCompile_TraversalFileID_SanitizedAgainstEscape(t *testing.T) { dlSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("fake-exmodz-bytes")) })) @@ -93,21 +94,22 @@ func TestDownloadMod_DeployCompile_TraversalFileName_SanitizedAgainstEscape(t *t require.NoError(t, svc.AddGame(game)) mod := &domain.Mod{ID: "bear-mount", SourceID: "fake-compiler", GameID: "icarus", Version: "3.3"} - file := &domain.DownloadableFile{ID: "exmodz", FileName: "../evil-traversal.exmodz"} + file := &domain.DownloadableFile{ID: "../evil-traversal-id", FileName: "Bear_Mount.exmodz"} result, err := svc.DownloadMod(context.Background(), "fake-compiler", game, mod, file, nil) require.NoError(t, err) - require.Equal(t, 1, result.FilesExtracted) + require.Equal(t, 0, result.FilesExtracted, "#197: DeployCompile ingest retains only, no per-mod deployment member") // The mod's own cache dir is cacheDir/icarus/fake-compiler-bear-mount/3.3 - // - an unsanitized "../evil-traversal" stem would climb into + // - an unsanitized "../evil-traversal-id" fileID would climb into // fake-compiler-bear-mount/ (one level up from the version dir). gameCache := svc.GetGameCache(game) - escapedPath := filepath.Join(gameCache.ModPath(game.ID, mod.SourceID, mod.ID, mod.Version), "..", "evil-traversal_P.pak") + escapedPath := filepath.Join(gameCache.ModPath(game.ID, mod.SourceID, mod.ID, mod.Version), "..", "evil-traversal-id") _, statErr := os.Stat(escapedPath) - require.True(t, os.IsNotExist(statErr), "a traversal filename's compiled output must never escape the version directory") + require.True(t, os.IsNotExist(statErr), "a traversal fileID's retained source must never escape the version directory") - files, err := gameCache.ListFiles(game.ID, mod.SourceID, mod.ID, mod.Version) - require.NoError(t, err) - require.Equal(t, []string{"evil-traversal_P.pak"}, files, "the sanitized (Base'd) compiled name is what must actually land in the cache") + retainedPath := gameCache.GetFilePath(game.ID, mod.SourceID, mod.ID, mod.Version, cache.RetainedSourceName(file.ID)) + data, err := os.ReadFile(retainedPath) + require.NoError(t, err, "the sanitized (Base'd) retained source must actually land inside the version directory") + require.Equal(t, "fake-exmodz-bytes", string(data)) } diff --git a/internal/core/updater.go b/internal/core/updater.go index a3753aa..3b4e65b 100644 --- a/internal/core/updater.go +++ b/internal/core/updater.go @@ -4,13 +4,9 @@ import ( "context" "errors" "fmt" - "io/fs" - "os" - "path/filepath" "github.com/DonovanMods/linux-mod-manager/internal/domain" "github.com/DonovanMods/linux-mod-manager/internal/source" - "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" ) // Updater checks for and applies mod updates @@ -152,299 +148,38 @@ func IsNewerVersion(currentVersion, newVersion string) bool { return domain.IsNewerVersion(currentVersion, newVersion) } -// CheckBaseStaleness scans installed for DeployCompile mods whose compiled -// artifact no longer matches game's live base data.pak IndexHash (#196, -// "the Friday problem": a weekly base-pak refresh silently reverts the -// tables a compiled mod patches, and nothing before #196 ever noticed). -// This is entirely local/offline - unlike Updater.CheckUpdates it never -// contacts a source, so it runs for EVERY installed mod regardless of -// UpdatePolicy or SourceID, including pinned and domain.SourceLocal mods -// (pinning fixes the mod version, not the base pak; a pure local import has -// no remote to check version-wise but can still go stale against the base -// pak - see #196 design point 3). -// -// Only entries carrying the #196 fingerprint (cache.BaseIndexHashes) -// participate: a missing fingerprint is treated as "not a compiled entry we -// can reason about", not "stale" - a game whose DeployMode is DeployCompile -// can still legitimately serve prebuilt, never-compiled .pak files (see -// isExmodzFile's doc comment), and there is no local signal to tell those -// apart from a compiled entry that merely predates #196. (Design point 4's -// literal "missing fingerprint = always stale" was deliberately narrowed -// during #196 review to avoid exactly that false positive - see the PR -// description.) Any mod actually compiled under #196 always carries the -// fingerprint by construction (stageCompileFingerprint writes it in the -// SAME atomic commit as the compiled pak), so this narrowing costs nothing -// for anything compiled going forward. -func (s *Service) CheckBaseStaleness(game *domain.Game, installed []domain.InstalledMod) ([]domain.Update, error) { - if game.DeployMode != domain.DeployCompile { - return nil, nil - } - basePakPath, err := resolveBasePak(game) - if err != nil { - // No installed base pak to compare against: nothing to report this - // pass, not an error - matches CheckUpdates' own per-source - // tolerance for a signal that simply isn't available right now. - return nil, nil //nolint:nilerr - } - liveHash, err := basePakIndexHash(basePakPath) - if err != nil { - return nil, fmt.Errorf("reading base pak for staleness check: %w", err) - } - - gameCache := s.GetGameCache(game) - var stale []domain.Update - for _, mod := range installed { - hashes, err := gameCache.BaseIndexHashes(game.ID, mod.SourceID, mod.ID, mod.Version) - if err != nil { - continue // unreadable bookkeeping: silently skip, matching FileManifests' own tolerance - } - for _, recorded := range hashes { - if recorded != liveHash { - stale = append(stale, domain.Update{InstalledMod: mod, NewVersion: mod.Version, RecompileNeeded: true}) - break - } - } - } - return stale, nil -} - // CheckGameUpdates is the single seam CLI and TUI both check updates -// through (#196): it combines Updater.CheckUpdates' remote version checks -// with CheckBaseStaleness' local base-pak staleness scan, so "does this mod -// need attention" means the same thing in both interfaces. A mod that -// already has a real version update available is not separately reported -// as stale even if it is: applying that update recompiles it fresh against -// the CURRENT base pak as a normal side effect of the compile step, so -// there is nothing left to flag once the real update lands. +// through (#196/#197): it combines Updater.CheckUpdates' remote version +// checks with CheckMergedPakStaleness' local merged-pak staleness scan +// (#197's generalization of #196's per-mod base-pak check to the merged +// model), so "does this profile need attention" means the same thing in +// both interfaces. profileName is required (#197): staleness is scoped to +// ONE profile's merged pak, not the whole game. // // Errors from either half are tolerated the same way CheckUpdates already // tolerates a single source failing: whatever updates were found are still // returned, with the first non-nil error surfaced (checkErr takes priority // as the richer, multi-source diagnostic when both fail). -func (s *Service) CheckGameUpdates(ctx context.Context, game *domain.Game, installed []domain.InstalledMod) ([]domain.Update, error) { +func (s *Service) CheckGameUpdates(ctx context.Context, game *domain.Game, profileName string, installed []domain.InstalledMod) ([]domain.Update, error) { updates, checkErr := s.NewUpdater().CheckUpdates(ctx, game, installed) - stale, staleErr := s.CheckBaseStaleness(game, installed) + staleUpd, staleErr := s.CheckMergedPakStaleness(game, profileName) if staleErr != nil && checkErr == nil { checkErr = staleErr } - if len(stale) > 0 { - reported := make(map[string]bool, len(updates)) + if staleUpd != nil { + reported := false for _, u := range updates { - reported[domain.ModKey(u.InstalledMod.SourceID, u.InstalledMod.ID)] = true - } - for _, u := range stale { - key := domain.ModKey(u.InstalledMod.SourceID, u.InstalledMod.ID) - if !reported[key] { - updates = append(updates, u) - reported[key] = true - } - } - } - - return updates, checkErr -} - -// ApplyRecompile recompiles mod IN PLACE at its CURRENT version against -// game's live base pak (#196: a base data.pak refresh left its already- -// deployed compile(s) patching stale tables - "the Friday problem"). -// Every compiled entry recorded for mod (cache.BaseIndexHashes) is -// recompiled, not just the ones CheckBaseStaleness found mismatched - a -// mod's compiled files always share one base pak, so a partial refresh -// would leave the entry internally inconsistent for no benefit. -// -// Recompiles from each file's retained .exmodz (offline) when present; -// falls back to re-downloading it from mod's source when the retained copy -// is missing AND a real fileID/source connection exists (a download- -// compiled entry). An import-compiled entry has no such connection (Import -// resolves no real source file ID - see stageCompileFingerprint) and a -// domain.SourceLocal mod has no source at all either way: for both, a -// missing retained source fails loud naming the fix (re-import/re-add the -// mod) rather than guessing. -// -// Locked mods are refused outright before any work happens - mirrors -// ApplyUpdate's own lock gate exactly (lock-wins: recompiling still -// rewrites the mod's deployed FILES even though its Version doesn't move, -// which is exactly what a lock forbids). Pinned mods ARE recompiled - -// pinning fixes the mod's VERSION, not the base pak (#196 design point 3); -// callers must not route a pinned mod through this gate at all. -// -// The cache update is staged and committed atomically (this package's own -// staging/commit pattern - prepareStaging + commitStagedCache), so a -// mid-recompile failure never touches the existing good entry. Redeploy -// reuses Installer.ReplaceForUpdate with the mod's OWN file IDs on both -// sides of the transition: with nothing IDs-wise changing, it degrades to -// its historical union-replace behavior, which simply re-links/re-copies -// every current member - exactly what refreshes a non-symlink deployment's -// stale on-disk bytes (a symlink deployment already reflects the new cache -// content once the atomic swap above lands, so this step is a correctness -// no-op for it, not a wasted one). -// -// ClassifyRetainedSourceStatError interprets os.Stat's error on a retained -// compile source path (#196 review): only a genuine "not exist" means -// missing (ok=true, err=nil) and falls through to the redownload/local- -// fails-loud logic below. Any OTHER stat error - permission denied, an I/O -// error, ... - is NOT "missing": folding it into the same code path would -// misreport a real filesystem problem as "re-import to restore it" or -// silently trigger a redownload the retained file didn't actually warrant. -// Such an error is returned instead, wrapped with %w so callers/tests can -// still errors.Is/As through to the original. Exported so it can be unit -// tested directly (deps/cache.go-style: this is a pure classifier, not a -// filesystem operation). -func ClassifyRetainedSourceStatError(statErr error) (missing bool, err error) { - if statErr == nil { - return false, nil - } - if errors.Is(statErr, fs.ErrNotExist) { - return true, nil - } - return false, statErr -} -func (s *Service) ApplyRecompile(ctx context.Context, game *domain.Game, profileName string, mod domain.InstalledMod, progress func(DeployProgress)) (result *UpdateApplyResult, err error) { - result = &UpdateApplyResult{} - emit := func(p DeployProgress) { - if progress != nil { - progress(p) - } - } - base := DeployProgress{ModName: mod.Name, ModID: mod.ID, SourceID: mod.SourceID} - - if prof, perr := s.NewProfileManager().Get(game.ID, profileName); perr == nil { - if ref := prof.FindRef(mod.SourceID, mod.ID); ref != nil && ref.Locked { - return result, LockedRefRefusalError(mod.Mod, profileName, ref) - } - } - - basePakPath, err := resolveBasePak(game) - if err != nil { - return result, err - } - - gameCache := s.GetGameCache(game) - hashes, err := gameCache.BaseIndexHashes(game.ID, mod.SourceID, mod.ID, mod.Version) - if err != nil { - return result, fmt.Errorf("reading compile fingerprints: %w", err) - } - if len(hashes) == 0 { - return result, fmt.Errorf("%s has no compiled entries to recompile", mod.Name) - } - - compiler, err := s.compilerSourceForGame(game.ID) - if err != nil { - return result, err - } - - manifests, err := gameCache.FileManifests(game.ID, mod.SourceID, mod.ID, mod.Version) - if err != nil { - return result, fmt.Errorf("reading cache manifests: %w", err) - } - - cacheModRef := &domain.Mod{ID: mod.ID, SourceID: mod.SourceID, Version: mod.Version, GameID: game.ID} - cachePath, stagePath, err := prepareStaging(gameCache, game, cacheModRef) - if err != nil { - return result, err - } - defer os.RemoveAll(stagePath) //nolint:errcheck - if err := os.MkdirAll(stagePath, 0755); err != nil { - return result, fmt.Errorf("preparing recompile staging: %w", err) - } - - // Lazily fetched: the common path (retained source present for every - // compiled file) never needs the source's file listing at all. - var sourceFiles []domain.DownloadableFile - var sourceFilesErr error - getSourceFiles := func() ([]domain.DownloadableFile, error) { - if sourceFiles == nil && sourceFilesErr == nil { - sourceFiles, sourceFilesErr = s.GetModFiles(ctx, mod.SourceID, &mod.Mod) - } - return sourceFiles, sourceFilesErr - } - - for fileID := range hashes { - // destName is the compiled output's own filename: for a download- - // compiled entry it's the recorded manifest member; for an import- - // compiled entry (no manifest - see importer.go's compile branch) - // fileID already IS that name (stageCompileFingerprint's doc - // comment), so the fallback is exact, not a guess. - destName := fileID - if m, ok := manifests[fileID]; ok && m.Recorded && len(m.Members) == 1 { - destName = m.Members[0] - } - - retainedPath := gameCache.GetFilePath(game.ID, mod.SourceID, mod.ID, mod.Version, cache.RetainedSourceName(fileID)) - sourcePath := retainedPath - _, statErr := os.Stat(retainedPath) - missing, statErr := ClassifyRetainedSourceStatError(statErr) - if statErr != nil { - return result, fmt.Errorf("%s: checking retained compile source for %q: %w", mod.Name, fileID, statErr) - } - if missing { - if mod.SourceID == domain.SourceLocal { - return result, fmt.Errorf("%s: retained compile source for %q is missing and this mod has no remote source to re-download from - re-import the .exmodz to restore it", mod.Name, fileID) - } - files, ferr := getSourceFiles() - if ferr != nil { - return result, fmt.Errorf("%s: retained compile source for %q is missing; fetching source files: %w", mod.Name, fileID, ferr) - } - var match *domain.DownloadableFile - for i := range files { - if files[i].ID == fileID { - match = &files[i] - break - } - } - if match == nil { - return result, fmt.Errorf("%s: retained compile source for %q is missing and no matching source file was found to re-download", mod.Name, fileID) - } - url, uerr := s.GetDownloadURL(ctx, mod.SourceID, &mod.Mod, match.ID) - if uerr != nil { - return result, fmt.Errorf("%s: re-downloading compile source: %w", mod.Name, uerr) - } - tempDir, terr := newStagingDir(s.stagingRoot(), "lmm-recompile-*") - if terr != nil { - return result, terr - } - defer os.RemoveAll(tempDir) //nolint:errcheck - // filepath.Base: match.FileName is source-controlled (a - // DownloadableFile from mod.SourceID's own listing) and must - // not be trusted as a path component verbatim - an entry like - // "../../evil.exmodz" would otherwise escape tempDir (#196 - // review). Same sanitization idiom already used elsewhere in - // this package for a source-derived name (importer.go's - // filepath.Base(archivePath), service.go's - // filepath.Base(localPath) fallback). - dlPath := filepath.Join(tempDir, filepath.Base(match.FileName)) - evt := base - evt.Phase, evt.Detail = UpdateNote, fmt.Sprintf("retained compile source missing for %s - re-downloading", destName) - emit(evt) - if _, derr := s.downloader.DownloadWithHeaders(ctx, url, dlPath, nil, nil); derr != nil { - return result, fmt.Errorf("%s: re-downloading compile source: %w", mod.Name, derr) + if u.InstalledMod.SourceID == staleUpd.InstalledMod.SourceID && u.InstalledMod.ID == staleUpd.InstalledMod.ID { + reported = true + break } - sourcePath = dlPath } - - outPath := filepath.Join(stagePath, destName) - if cerr := compiler.Compile(ctx, basePakPath, sourcePath, outPath); cerr != nil { - return result, fmt.Errorf("recompiling %s: %w", destName, cerr) - } - if ferr := stageCompileFingerprint(stagePath, fileID, basePakPath, sourcePath); ferr != nil { - return result, ferr + if !reported { + updates = append(updates, *staleUpd) } - result.Applied = append(result.Applied, destName) - } - - if err := commitStagedCache(cachePath, stagePath); err != nil { - return result, err } - installer, err := s.GetInstallerForProfile(game, profileName) - if err != nil { - return result, fmt.Errorf("recompiled %s but could not redeploy: %w", mod.Name, err) - } - if err := installer.ReplaceForUpdate(ctx, game, &mod.Mod, &mod.Mod, profileName, mod.FileIDs, mod.FileIDs); err != nil { - return result, fmt.Errorf("recompiled %s but redeploying failed: %w", mod.Name, err) - } - - return result, nil + return updates, checkErr } diff --git a/internal/storage/cache/cache.go b/internal/storage/cache/cache.go index 2bd7019..83b64be 100644 --- a/internal/storage/cache/cache.go +++ b/internal/storage/cache/cache.go @@ -323,9 +323,16 @@ const retainedSourcePrefix = ReservedPrefix + "source-" // retained compile source. It is a pure naming function - like // GetFilePath, callers join it against a staging or cache directory // themselves and read/write/copy the actual bytes with ordinary file I/O -// (see internal/core's stageCompileFingerprint and recompile-apply path). +// (see internal/core's ingest/merge paths). +// +// fileID is Base'd first (#197 hardening): it is source-controlled (a +// ModSource's own DownloadableFile.ID, or - for an import - the archive's +// own filename) exactly like the FileName fields #196's review already +// found needed filepath.Base sanitization at their own join sites - a +// fileID containing "../" must not be able to escape the staging/cache +// directory this name gets joined against downstream. func RetainedSourceName(fileID string) string { - return retainedSourcePrefix + fileID + return retainedSourcePrefix + filepath.Base(fileID) } // mergeFingerprintMarkerName names the single JSON fingerprint marker a From a497ab96c78896e24b3c1356c69c81ee2c23530f Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 21:05:13 -0400 Subject: [PATCH 71/96] feat: cmd/lmm targets merged-pak regen/staleness instead of per-mod recompile (#197) --- cmd/lmm/install.go | 2 +- cmd/lmm/install_compile_test.go | 59 ++++++++++++++++++++------------ cmd/lmm/profile.go | 2 -- cmd/lmm/update.go | 34 +++++++++--------- cmd/lmm/update_recompile_test.go | 56 ++++++++---------------------- cmd/lmm/verify.go | 37 ++++++++------------ cmd/lmm/verify_recompile_test.go | 4 ++- internal/core/flows.go | 34 ++++++++++-------- 8 files changed, 108 insertions(+), 120 deletions(-) diff --git a/cmd/lmm/install.go b/cmd/lmm/install.go index 7b15423..277f725 100644 --- a/cmd/lmm/install.go +++ b/cmd/lmm/install.go @@ -615,7 +615,7 @@ func doInstall(ctx context.Context, service *core.Service, game *domain.Game, ar case core.InstallChecksumComputed: fmt.Printf(" Checksum: %s\n", truncateChecksum(p.Detail)) case core.InstallCompiling: - fmt.Printf("\nCompiling %s → %s...\n", displayFileLabel(*p.File), p.Detail) + fmt.Printf("\nRetaining %s for merge...\n", displayFileLabel(*p.File)) case core.InstallExtracting: fmt.Println("\nExtracting to cache...") case core.InstallDeploying: diff --git a/cmd/lmm/install_compile_test.go b/cmd/lmm/install_compile_test.go index 1c53202..15c976b 100644 --- a/cmd/lmm/install_compile_test.go +++ b/cmd/lmm/install_compile_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/source" "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -23,35 +24,51 @@ func writeFakeBasePak(t *testing.T, path string) { require.NoError(t, w.Close()) } -// compilerInstallSource wraps fakeInstallSource with a source.Compiler +// compilerInstallSource wraps fakeInstallSource with a source.MergeCompiler // implementation, so `lmm install` can drive a real DeployCompile game // end-to-end through the CLI's exact console-output path (mirrors // internal/core/service_icarus_compile_test.go's fakeCompilerSource, at the // CLI layer instead of core's). type compilerInstallSource struct { *fakeInstallSource - compileCalls int + validateCalls int + compileCalls int } -// Compile copies the downloaded source file through unchanged - this test -// only asserts the CLI announces the compile step and uses its output, not -// that real PAK compilation happens (internal/unrealpak's own tests cover -// that). -func (s *compilerInstallSource) Compile(ctx context.Context, basePakPath, sourceFilePath, outputPath string) error { +// ValidateSource confirms the archive exists - this test only asserts the +// CLI announces the retain step, not that real .exmodz parsing happens +// (internal/source/icarus's own tests cover that). +func (s *compilerInstallSource) ValidateSource(sourceFilePath string) error { + s.validateCalls++ + _, err := os.Stat(sourceFilePath) + return err +} + +// MergeCompile concatenates every source's bytes - enough for tests to +// prove a merge/regen actually happened and used the retained content, +// without needing a real base pak table to patch (mirrors +// internal/core/service_icarus_compile_test.go's fakeCompilerSource). +func (s *compilerInstallSource) MergeCompile(ctx context.Context, basePakPath string, sources []source.MergeSource, outputPath string) ([]string, error) { s.compileCalls++ - data, err := os.ReadFile(sourceFilePath) - if err != nil { - return err + var out []byte + for _, src := range sources { + data, err := os.ReadFile(src.ExmodzPath) + if err != nil { + return nil, err + } + out = append(out, data...) } - return os.WriteFile(outputPath, data, 0o644) + return nil, os.WriteFile(outputPath, out, 0o644) } -// TestDoInstall_DeployCompile_AnnouncesCompiling guards #190 item 1: an -// install that compiles a .exmodz file must announce the compile step by -// name, not the generic "Extracting to cache..." line the plain -// extract/copy path uses (which is actively misleading here - compiling -// isn't extracting). -func TestDoInstall_DeployCompile_AnnouncesCompiling(t *testing.T) { +// TestDoInstall_DeployCompile_AnnouncesRetaining guards #190 item 1: an +// install that ingests an ".exmodz" file must announce the retain-for-merge +// step by name, not the generic "Extracting to cache..." line the plain +// extract/copy path uses (which is actively misleading here - nothing is +// extracted). #197: ingest validates+retains only, so this test's premise +// changed from "announces compiling" to "announces retaining" - the actual +// merge is batched across the whole profile and happens later. +func TestDoInstall_DeployCompile_AnnouncesRetaining(t *testing.T) { svc, game, src := setupDoInstallTest(t) game.DeployMode = domain.DeployCompile game.InstallPath = t.TempDir() @@ -62,7 +79,7 @@ func TestDoInstall_DeployCompile_AnnouncesCompiling(t *testing.T) { compiler := &compilerInstallSource{fakeInstallSource: src} // Re-register under the same ID so doInstall's resolved source is the - // compiler-capable wrapper, not the plain fake registered by + // merge-compiler-capable wrapper, not the plain fake registered by // setupDoInstallTest. svc.RegisterSource(compiler) @@ -74,7 +91,7 @@ func TestDoInstall_DeployCompile_AnnouncesCompiling(t *testing.T) { return doInstall(context.Background(), svc, game, nil) }) - assert.Equal(t, 1, compiler.compileCalls) - assert.Contains(t, out, "Compiling Bear_Mount.exmodz → Bear_Mount_P.pak...\n") - assert.NotContains(t, out, "Extracting to cache...", "compiling isn't extracting - the generic message must not also print") + assert.Equal(t, 1, compiler.validateCalls) + assert.Contains(t, out, "Retaining Bear_Mount.exmodz for merge...\n") + assert.NotContains(t, out, "Extracting to cache...", "retaining isn't extracting - the generic message must not also print") } diff --git a/cmd/lmm/profile.go b/cmd/lmm/profile.go index 15b390c..8d139c4 100644 --- a/cmd/lmm/profile.go +++ b/cmd/lmm/profile.go @@ -742,8 +742,6 @@ func doProfileReorder(service *core.Service, game *domain.Game, args []string) e return fmt.Errorf("loading profile: %w", err) } - pm := getProfileManager(service) - if len(args) == 0 { // Show current load order if len(profile.Mods) == 0 { diff --git a/cmd/lmm/update.go b/cmd/lmm/update.go index 0024932..f5ee7cc 100644 --- a/cmd/lmm/update.go +++ b/cmd/lmm/update.go @@ -314,9 +314,9 @@ func doUpdate(ctx context.Context, service *core.Service, game *domain.Game, arg } // Check for updates (partial results returned even when some mods fail to - // fetch) plus, for DeployCompile games, base-pak staleness (#196) - + // fetch) plus, for DeployCompile games, merged-pak staleness (#196/#197) - // CheckGameUpdates is the single seam CLI and TUI both check through. - updates, checkErr := service.CheckGameUpdates(ctx, game, installed) + updates, checkErr := service.CheckGameUpdates(ctx, game, profileName, installed) if checkErr != nil { if errors.Is(checkErr, domain.ErrAuthRequired) { return authPromptError(updateSource) @@ -597,8 +597,8 @@ func applySingleUpdate(ctx context.Context, service *core.Service, game *domain. } } - // Check for update for this specific mod (plus base-pak staleness, #196) - updates, err := service.CheckGameUpdates(ctx, game, []domain.InstalledMod{*mod}) + // Check for update for this specific mod (plus merged-pak staleness, #196/#197) + updates, err := service.CheckGameUpdates(ctx, game, profileName, []domain.InstalledMod{*mod}) if err != nil { if errors.Is(err, domain.ErrAuthRequired) { return authPromptError(updateSource) @@ -669,7 +669,7 @@ func applySingleUpdate(ctx context.Context, service *core.Service, game *domain. return nil } - if err := applyRecompile(ctx, service, game, *mod, profileName); err != nil { + if err := applyRecompile(ctx, service, game, profileName); err != nil { return err } @@ -759,13 +759,13 @@ func applySingleUpdate(ctx context.Context, service *core.Service, game *domain. // exact console positioning (download progress, forced-hook warnings, // after_each hook warnings, and the --verbose-gated link-method note). // -// #196: a RecompileNeeded row carries no real version change (NewVersion == -// InstalledMod.Version) - it is routed to Service.ApplyRecompile instead, -// which has no hooks to run and no version/FileIDs to record, only the -// recompile-and-redeploy step itself. +// #196/#197: a RecompileNeeded row carries no real version change +// (NewVersion == InstalledMod.Version) - it is routed to +// Service.ApplyMergedPakRegen instead, which has no hooks to run and no +// version/FileIDs to record, only the merge-and-redeploy step itself. func applyUpdate(ctx context.Context, service *core.Service, game *domain.Game, upd domain.Update, profileName string) error { if upd.RecompileNeeded { - return applyRecompile(ctx, service, game, upd.InstalledMod, profileName) + return applyRecompile(ctx, service, game, profileName) } opts := core.UpdateOptions{ @@ -798,12 +798,12 @@ func applyUpdate(ctx context.Context, service *core.Service, game *domain.Game, return err } -// applyRecompile applies a #196 base-pak staleness row via -// Service.ApplyRecompile, printing from its progress events the same way -// applyUpdate does for its own (UpdateWarning/UpdateNote are the only -// phases ApplyRecompile emits - it runs no hooks and downloads nothing -// worth a progress bar). -func applyRecompile(ctx context.Context, service *core.Service, game *domain.Game, mod domain.InstalledMod, profileName string) error { +// applyRecompile applies a #197 merged-pak staleness row via +// Service.ApplyMergedPakRegen, printing from its progress events the same +// way applyUpdate does for its own (UpdateWarning/UpdateNote are the only +// phases ApplyMergedPakRegen emits - it runs no hooks and downloads +// nothing worth a progress bar). +func applyRecompile(ctx context.Context, service *core.Service, game *domain.Game, profileName string) error { progress := func(p core.DeployProgress) { switch p.Phase { case core.UpdateWarning: @@ -815,7 +815,7 @@ func applyRecompile(ctx context.Context, service *core.Service, game *domain.Gam } } - _, err := service.ApplyRecompile(ctx, game, profileName, mod, progress) + _, err := service.ApplyMergedPakRegen(ctx, game, profileName, progress) return err } diff --git a/cmd/lmm/update_recompile_test.go b/cmd/lmm/update_recompile_test.go index 29018a5..4c91f19 100644 --- a/cmd/lmm/update_recompile_test.go +++ b/cmd/lmm/update_recompile_test.go @@ -16,12 +16,12 @@ import ( ) // setupDoUpdateRecompileTest builds a DeployCompile game with a registered -// compiler-capable source and an installed, deployed compiled mod whose -// recorded base-pak fingerprint is deliberately wrong, so `lmm update` -// reports/applies a #196 recompile row end to end through the CLI. -// linkMethod is LinkCopy so a successful recompile+redeploy is provable -// from the on-disk deployed bytes (a symlink would trivially reflect an -// in-place cache swap on its own). +// merge-compiler-capable source and an ENABLED exmodz mod, deliberately +// leaving the merged pak un-generated so `lmm update` reports/applies a +// #197 merge-needed row end to end through the CLI. linkMethod is LinkCopy +// so a successful regen+redeploy is provable from the on-disk deployed +// bytes (a symlink would trivially reflect an in-place cache swap on its +// own). func setupDoUpdateRecompileTest(t *testing.T) (*core.Service, *domain.Game, *compilerInstallSource, string) { t.Helper() @@ -64,31 +64,23 @@ func setupDoUpdateRecompileTest(t *testing.T) (*core.Service, *domain.Game, *com const modID, version, fileID = "bear-mount", "3.3", "exmodz-file-id" gameCache := svc.GetGameCache(game) - require.NoError(t, gameCache.Store(game.ID, "fake-compiler", modID, version, "Bear_Mount_P.pak", []byte("stale-compiled-bytes"))) require.NoError(t, gameCache.Store(game.ID, "fake-compiler", modID, version, cache.RetainedSourceName(fileID), []byte("retained-exmodz-bytes"))) - versionDir := gameCache.ModPath(game.ID, "fake-compiler", modID, version) - require.NoError(t, cache.MarkFileCompleteWithMembers(versionDir, fileID, []string{"Bear_Mount_P.pak"})) - require.NoError(t, cache.MarkBaseIndexHash(versionDir, fileID, "0000000000000000000000000000000000dead")) im := &domain.InstalledMod{ Mod: domain.Mod{ID: modID, SourceID: "fake-compiler", Name: "Bear Mount", Version: version, GameID: game.ID}, ProfileName: "default", UpdatePolicy: domain.UpdateNotify, Enabled: true, - Deployed: true, - LinkMethod: domain.LinkCopy, FileIDs: []string{fileID}, } require.NoError(t, svc.SaveInstalledMod(im)) - installer := svc.GetInstaller(game) - require.NoError(t, installer.Install(context.Background(), game, &im.Mod, "default")) pm := svc.NewProfileManager() _, cerr := pm.Create(game.ID, "default") require.NoError(t, cerr) require.NoError(t, pm.UpsertMod(game.ID, "default", domain.ModReference{SourceID: "fake-compiler", ModID: modID, Version: version, FileIDs: []string{fileID}})) - return svc, game, compiler, filepath.Join(game.ModPath, "Bear_Mount_P.pak") + return svc, game, compiler, filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") } // TestDoUpdate_JSON_ReportsRecompileNeeded proves the bulk --json contract @@ -113,9 +105,13 @@ func TestDoUpdate_JSON_ReportsRecompileNeeded(t *testing.T) { require.NoError(t, json.Unmarshal(buf.Bytes(), &out)) require.Len(t, out.Updates, 1) row := out.Updates[0] - assert.Equal(t, "bear-mount", row.ModID) - assert.Equal(t, "3.3", row.Current) - assert.Equal(t, "3.3", row.Available, "a staleness row's available version equals current - the mod hasn't changed") + // #197: a merged-pak staleness row's identity is the SYNTHETIC + // merged-pak mod (domain.SourceMerged/"merged-pak"), not the + // contributing "bear-mount" mod - CheckMergedPakStaleness is + // profile-scoped, not per-mod. + assert.Equal(t, "merged-pak", row.ModID) + assert.Equal(t, "merged", row.Current) + assert.Equal(t, "merged", row.Available, "a staleness row's available version equals current - nothing has a real version change") assert.True(t, row.RecompileNeeded) assert.Equal(t, "stale_compile", row.Reason) } @@ -165,27 +161,3 @@ func TestApplySingleUpdate_Recompile_JSON(t *testing.T) { assert.Equal(t, "3.3", out.FromVersion) assert.Equal(t, "3.3", out.ToVersion) } - -// TestApplySingleUpdate_Recompile_LockedRefuses proves the CLI's locked- -// refusal wording fires for a staleness row too, and never touches the -// locked mod's deployed files. -func TestApplySingleUpdate_Recompile_LockedRefuses(t *testing.T) { - svc, game, compiler, deployedPath := setupDoUpdateRecompileTest(t) - - pm := svc.NewProfileManager() - require.NoError(t, pm.SetModLock(game.ID, "default", "fake-compiler", "bear-mount", "")) - - before, err := os.ReadFile(deployedPath) - require.NoError(t, err) - - mod, err := svc.GetInstalledMod("fake-compiler", "bear-mount", "icarus", "default") - require.NoError(t, err) - - err = applySingleUpdate(context.Background(), svc, game, mod, "default") - require.NoError(t, err, "a locked skip is reported, not returned as an error") - assert.Equal(t, 0, compiler.compileCalls, "a locked mod must never be recompiled") - - after, err := os.ReadFile(deployedPath) - require.NoError(t, err) - assert.Equal(t, before, after, "a locked mod's deployed files must never be touched") -} diff --git a/cmd/lmm/verify.go b/cmd/lmm/verify.go index 06a3862..271002a 100644 --- a/cmd/lmm/verify.go +++ b/cmd/lmm/verify.go @@ -306,37 +306,30 @@ func doVerify(cmd *cobra.Command, svc *core.Service, game *domain.Game, args []s return fmt.Errorf("getting installed mods: %w", err) } - // Base-pak staleness check (#196): for a DeployCompile game, compare - // each compiled mod's recorded base-pak fingerprint against the game's - // live base pak. Entirely local/offline - unlike the version-record - // check below, this is NOT skipped for local-source or manual-download - // mods (there is no source dependency at all; see - // Service.CheckBaseStaleness's own doc comment). + // Merged-pak staleness check (#197, generalizing #196's per-mod + // version): for a DeployCompile game, compare the profile's merged + // pak's recorded fingerprint against the game's CURRENT enabled-mod + // set/order/versions/base pak. Entirely local/offline. modFilter has no + // effect here - the merged pak is profile-scoped, not per-mod, so + // `lmm verify ` still checks it (a single mod's own version + // mismatch and the profile's overall merge staleness are independent + // facts). if game.DeployMode == domain.DeployCompile { - staleCheckSet := installedMods - if modFilter != "" { - staleCheckSet = nil - for i := range installedMods { - if installedMods[i].ID == modFilter { - staleCheckSet = append(staleCheckSet, installedMods[i]) - } - } - } - stale, serr := svc.CheckBaseStaleness(game, staleCheckSet) + staleUpd, serr := svc.CheckMergedPakStaleness(game, profile) if serr != nil { if jsonOutput { - jsonFiles = append(jsonFiles, verifyFileJSON{Status: "skipped", Note: fmt.Sprintf("could not check base pak staleness: %v", serr)}) + jsonFiles = append(jsonFiles, verifyFileJSON{Status: "skipped", Note: fmt.Sprintf("could not check merged pak staleness: %v", serr)}) } else { - fmt.Printf("%s could not check base pak staleness: %v\n", colorYellow("?"), serr) + fmt.Printf("%s could not check merged pak staleness: %v\n", colorYellow("?"), serr) } warnings++ } - checked += len(staleCheckSet) - for _, u := range stale { + checked++ + if staleUpd != nil { if jsonOutput { - jsonFiles = append(jsonFiles, verifyFileJSON{ModID: u.InstalledMod.ID, ModName: u.InstalledMod.Name, Status: "stale_compile"}) + jsonFiles = append(jsonFiles, verifyFileJSON{ModID: staleUpd.InstalledMod.ID, ModName: staleUpd.InstalledMod.Name, Status: "stale_compile"}) } else { - fmt.Printf("%s %s - RECOMPILE NEEDED (base pak updated - run 'lmm update' to fix)\n", colorYellow("?"), u.InstalledMod.Name) + fmt.Printf("%s %s - RECOMPILE NEEDED (base pak updated - run 'lmm update' to fix)\n", colorYellow("?"), staleUpd.InstalledMod.Name) } warnings++ } diff --git a/cmd/lmm/verify_recompile_test.go b/cmd/lmm/verify_recompile_test.go index e711a38..f4082fe 100644 --- a/cmd/lmm/verify_recompile_test.go +++ b/cmd/lmm/verify_recompile_test.go @@ -73,6 +73,8 @@ func TestDoVerify_StaleCompile_JSON(t *testing.T) { } } require.NotNil(t, found, "expected a stale_compile row") - assert.Equal(t, "bear-mount", found.ModID) + // #197: a merged-pak staleness row's identity is the SYNTHETIC + // merged-pak mod, not the contributing "bear-mount" mod. + assert.Equal(t, "merged-pak", found.ModID) assert.GreaterOrEqual(t, out.Warnings, 1) } diff --git a/internal/core/flows.go b/internal/core/flows.go index efe20db..3da1166 100644 --- a/internal/core/flows.go +++ b/internal/core/flows.go @@ -731,14 +731,16 @@ const ( // way; the CLI applies its own truncateChecksum. InstallChecksumComputed // InstallCompiling fires instead of InstallExtracting, once per file, - // when a DeployCompile game's ".exmodz" file was actually compiled - // (#190 item 1) - the generic "Extracting to cache..." wording is - // misleading for a compile step, which never extracts anything. File - // identifies the source file (for displayFileLabel); Detail carries the - // compiled output filename (e.g. "Bear_Mount_P.pak"), so the CLI can - // announce "Compiling → ..." without core owning the - // exact sentence. The BATCH path never prints this (it has no - // DeployCompile support and no equivalent status line at all). + // when a DeployCompile game's ".exmodz" file was validated and retained + // for a later merge (#190 item 1; #197: ingest no longer compiles a + // per-mod pak - the real merge happens once, batched across the whole + // profile, via Service.syncMergedPak) - the generic "Extracting to + // cache..." wording is misleading here either way, since nothing is + // extracted. File identifies the source file (for displayFileLabel); + // Detail is unset (there is no per-file compiled output filename left + // to announce under the merged-only model). The BATCH path never + // prints this (it has no DeployCompile support and no equivalent + // status line at all). InstallCompiling // InstallExtracting mirrors doInstall's unconditional "Extracting to // cache..." status line, fired once after the STRICT-path primary's @@ -4101,15 +4103,19 @@ func (s *Service) applyInstallPrimary(ctx context.Context, game *domain.Game, pl } } - // A compiled file was never "extracted" - announce the compile step by - // name instead of the generic message, which is actively misleading - // here (#190 item 1). Only fires for files that actually compiled, so - // every non-DeployCompile (or non-exmodz) install keeps today's exact - // "Extracting to cache..." text unchanged. + // A retained-for-merge file was never "extracted" - announce the step + // by name instead of the generic message, which is actively misleading + // here (#190 item 1). Only fires for files that actually go through + // ingest's validate+retain branch, so every non-DeployCompile (or + // non-exmodz) install keeps today's exact "Extracting to cache..." text + // unchanged. #197: this no longer compiles a per-mod pak - the merge + // happens later, batched across the whole profile, via + // Service.syncMergedPak - so there is no compiled output filename left + // to announce (Detail is unset). if len(compiledFiles) > 0 { for _, cf := range compiledFiles { evt := base - evt.Phase, evt.File, evt.Detail = InstallCompiling, cf, compiledFileName(cf.FileName) + evt.Phase, evt.File = InstallCompiling, cf emit(evt) } } else { From e4d6e212e557dcc8007c56164158504baec675f4 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 21:05:18 -0400 Subject: [PATCH 72/96] feat: internal/tui targets merged-pak regen/staleness instead of per-mod recompile (#197) --- internal/tui/service_core.go | 44 ++++++++----- internal/tui/service_core_internal_test.go | 5 +- internal/tui/service_core_recompile_test.go | 72 ++++++++++++--------- 3 files changed, 74 insertions(+), 47 deletions(-) diff --git a/internal/tui/service_core.go b/internal/tui/service_core.go index 420374e..0550b50 100644 --- a/internal/tui/service_core.go +++ b/internal/tui/service_core.go @@ -1147,7 +1147,7 @@ func installProgressLine(modName string, p core.DeployProgress) (ActionProgress, case core.InstallDepDownloading: return ActionProgress{Line: fmt.Sprintf("Installing %s: %.0f%%", p.ModName, p.Percent), Percent: p.Percent}, true case core.InstallCompiling: - return ActionProgress{Line: fmt.Sprintf("Installing %s: compiling", modName), Percent: -1}, true + return ActionProgress{Line: fmt.Sprintf("Installing %s: retaining", modName), Percent: -1}, true case core.InstallExtracting: return ActionProgress{Line: fmt.Sprintf("Installing %s: extracting", modName), Percent: -1}, true case core.InstallDeploying: @@ -1423,7 +1423,7 @@ func (p *coreProvider) CheckUpdates(ctx context.Context) (UpdatesView, error) { return UpdatesView{}, fmt.Errorf("loading installed mods for %s/%s: %w", game.ID, profile, err) } - updates, checkErr := p.svc.CheckGameUpdates(ctx, game, installed) + updates, checkErr := p.svc.CheckGameUpdates(ctx, game, profile, installed) // #143: join the profile YAML's lock state onto the update rows - the // same projection (and the same nil-safe "an unreadable profile leaves @@ -1472,18 +1472,39 @@ func (p *coreProvider) CheckUpdates(ctx context.Context) (UpdatesView, error) { // and a real update may need that superseded-file-ID mapping to install // correctly; only a fresh check call can supply it. // -// #196: a RecompileNeeded row (NewVersion == the mod's current version - a -// base-pak staleness signal, not a real update) is routed to -// Service.ApplyRecompile instead, which has no hooks/options to configure. +// #196/#197: a RecompileNeeded row (NewVersion == the mod's current +// version - a merged-pak staleness signal, not a real update) is routed to +// Service.ApplyMergedPakRegen instead, which has no hooks/options to +// configure. This check MUST happen before GetInstalledMod below: a +// RecompileNeeded row's u.Source/u.ID identify the SYNTHETIC merged-pak +// row (domain.SourceMerged/"merged-pak"), which has no real +// installed_mods DB row - GetInstalledMod would fail loud for it. (Caught +// while wiring this task: the original draft called GetInstalledMod +// unconditionally first, which broke exactly this case.) func (p *coreProvider) ApplyUpdate(ctx context.Context, u UpdateItem, progress func(ActionProgress)) (ActionOutcome, error) { game := p.currentGame() profile := p.currentProfile() + + if u.RecompileNeeded { + adapter := deployProgressAdapter(progress, func(p core.DeployProgress) (ActionProgress, bool) { + return updateProgressLine(u.Name, p) + }) + result, err := p.svc.ApplyMergedPakRegen(ctx, game, profile, adapter) + if err != nil { + return ActionOutcome{}, mapUpdateNetworkError(fmt.Sprintf("recompiling %s", u.Name), u.Source, err) + } + return ActionOutcome{ + Message: fmt.Sprintf("Recompiled %q (base pak updated)", u.Name), + Warnings: mergeDiagnostics(result.Warnings, result.Notes), + }, nil + } + mod, err := p.svc.GetInstalledMod(u.Source, u.ID, game.ID, profile) if err != nil { return ActionOutcome{}, fmt.Errorf("getting installed mod %s: %w", u.Name, err) } - updates, err := p.svc.CheckGameUpdates(ctx, game, []domain.InstalledMod{*mod}) + updates, err := p.svc.CheckGameUpdates(ctx, game, profile, []domain.InstalledMod{*mod}) if err != nil { return ActionOutcome{}, mapUpdateNetworkError(fmt.Sprintf("checking update for %s", u.Name), u.Source, err) } @@ -1496,17 +1517,6 @@ func (p *coreProvider) ApplyUpdate(ctx context.Context, u UpdateItem, progress f return updateProgressLine(u.Name, p) }) - if upd.RecompileNeeded { - result, err := p.svc.ApplyRecompile(ctx, game, profile, upd.InstalledMod, adapter) - if err != nil { - return ActionOutcome{}, mapUpdateNetworkError(fmt.Sprintf("recompiling %s", u.Name), u.Source, err) - } - return ActionOutcome{ - Message: fmt.Sprintf("Recompiled %q (base pak updated)", u.Name), - Warnings: mergeDiagnostics(result.Warnings, result.Notes), - }, nil - } - opts := core.UpdateOptions{ Hooks: p.resolvedHooks(game, profile), HookRunner: p.hookRunner(), diff --git a/internal/tui/service_core_internal_test.go b/internal/tui/service_core_internal_test.go index 46fd21e..991bcfa 100644 --- a/internal/tui/service_core_internal_test.go +++ b/internal/tui/service_core_internal_test.go @@ -52,9 +52,12 @@ func TestSwitchProgressLine_UnhandledPhaseStillDrops(t *testing.T) { // the TUI drives the exact same core.ApplyInstall/DeployProgress path as the // CLI (installProgressLine's own doc comment), so InstallCompiling must // compose a status line here too, not silently fall to the default case. +// #197: ingest no longer compiles a per-mod pak, so the line now says +// "retaining" (validated+retained for a later merge) instead of +// "compiling" - see Service.syncMergedPak. func TestInstallProgressLine_RendersCompiling(t *testing.T) { line, ok := installProgressLine("Bear Mount", core.DeployProgress{Phase: core.InstallCompiling}) assert.True(t, ok, "InstallCompiling must compose a visible progress line, not be dropped") assert.Contains(t, line.Line, "Bear Mount") - assert.Contains(t, line.Line, "compiling") + assert.Contains(t, line.Line, "retaining") } diff --git a/internal/tui/service_core_recompile_test.go b/internal/tui/service_core_recompile_test.go index e0e4c7a..e0ffe0d 100644 --- a/internal/tui/service_core_recompile_test.go +++ b/internal/tui/service_core_recompile_test.go @@ -15,12 +15,13 @@ import ( "github.com/stretchr/testify/require" ) -// recompileFakeSource is a minimal ModSource + source.Compiler standing in -// for internal/source/icarus.Icarus, mirroring +// recompileFakeSource is a minimal ModSource + source.MergeCompiler +// standing in for internal/source/icarus.Icarus, mirroring // internal/core/service_icarus_compile_test.go's fakeCompilerSource at the // TUI layer. type recompileFakeSource struct { - compileCalls int + validateCalls int + compileCalls int } func (s *recompileFakeSource) ID() string { return "fake-compiler" } @@ -47,24 +48,40 @@ func (s *recompileFakeSource) GetDownloadURL(ctx context.Context, mod *domain.Mo func (s *recompileFakeSource) CheckUpdates(ctx context.Context, installed []domain.InstalledMod) ([]domain.Update, error) { return nil, nil } -func (s *recompileFakeSource) Compile(ctx context.Context, basePakPath, sourceFilePath, outputPath string) error { + +// ValidateSource confirms the archive exists. +func (s *recompileFakeSource) ValidateSource(sourceFilePath string) error { + s.validateCalls++ + _, err := os.Stat(sourceFilePath) + return err +} + +// MergeCompile concatenates every source's bytes - enough to prove a +// merge/regen actually happened and used the retained content. +func (s *recompileFakeSource) MergeCompile(ctx context.Context, basePakPath string, sources []source.MergeSource, outputPath string) ([]string, error) { s.compileCalls++ - data, err := os.ReadFile(sourceFilePath) - if err != nil { - return err + var out []byte + for _, src := range sources { + data, err := os.ReadFile(src.ExmodzPath) + if err != nil { + return nil, err + } + out = append(out, data...) } - return os.WriteFile(outputPath, data, 0o644) + return nil, os.WriteFile(outputPath, out, 0o644) } var ( - _ source.ModSource = (*recompileFakeSource)(nil) - _ source.Compiler = (*recompileFakeSource)(nil) + _ source.ModSource = (*recompileFakeSource)(nil) + _ source.MergeCompiler = (*recompileFakeSource)(nil) ) -// newRecompileActionsFixture builds a DeployCompile game with an installed, -// deployed compiled mod whose recorded base-pak fingerprint is wrong, and -// returns the ActionProvider (#196's CLI/TUI parity seam), the fake -// compiler, and the deployed file's game-dir path. +// newRecompileActionsFixture builds a DeployCompile game with a registered +// merge-compiler-capable source and an ENABLED exmodz mod, deliberately +// leaving the merged pak un-generated so the TUI's update check reports/ +// applies a #197 merge-needed row end to end, returning the ActionProvider +// (#196's CLI/TUI parity seam), the fake compiler, and the merged pak's +// game-dir path. func newRecompileActionsFixture(t *testing.T) (tui.ActionProvider, *recompileFakeSource, string) { t.Helper() @@ -97,31 +114,23 @@ func newRecompileActionsFixture(t *testing.T) (tui.ActionProvider, *recompileFak const modID, version, fileID = "bear-mount", "3.3", "exmodz-file-id" gameCache := svc.GetGameCache(game) - require.NoError(t, gameCache.Store(game.ID, "fake-compiler", modID, version, "Bear_Mount_P.pak", []byte("stale-compiled-bytes"))) require.NoError(t, gameCache.Store(game.ID, "fake-compiler", modID, version, cache.RetainedSourceName(fileID), []byte("retained-exmodz-bytes"))) - versionDir := gameCache.ModPath(game.ID, "fake-compiler", modID, version) - require.NoError(t, cache.MarkFileCompleteWithMembers(versionDir, fileID, []string{"Bear_Mount_P.pak"})) - require.NoError(t, cache.MarkBaseIndexHash(versionDir, fileID, "0000000000000000000000000000000000dead")) im := &domain.InstalledMod{ Mod: domain.Mod{ID: modID, SourceID: "fake-compiler", Name: "Bear Mount", Version: version, GameID: game.ID}, ProfileName: "default", UpdatePolicy: domain.UpdateNotify, Enabled: true, - Deployed: true, - LinkMethod: domain.LinkCopy, FileIDs: []string{fileID}, } require.NoError(t, svc.SaveInstalledMod(im)) - installer := svc.GetInstaller(game) - require.NoError(t, installer.Install(context.Background(), game, &im.Mod, "default")) require.NoError(t, pm.UpsertMod(game.ID, "default", domain.ModReference{SourceID: "fake-compiler", ModID: modID, Version: version, FileIDs: []string{fileID}})) - return tui.NewCoreActions(svc, game, "default"), compiler, filepath.Join(game.ModPath, "Bear_Mount_P.pak") + return tui.NewCoreActions(svc, game, "default"), compiler, filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") } // TestCoreProviderActions_CheckUpdates_ReportsRecompileNeeded proves the TUI -// update check reports a #196 base-pak staleness row through the same +// update check reports a #197 merged-pak staleness row through the same // CheckGameUpdates seam the CLI uses. func TestCoreProviderActions_CheckUpdates_ReportsRecompileNeeded(t *testing.T) { actions, _, _ := newRecompileActionsFixture(t) @@ -130,16 +139,21 @@ func TestCoreProviderActions_CheckUpdates_ReportsRecompileNeeded(t *testing.T) { require.NoError(t, err) require.Len(t, view.Updates, 1) u := view.Updates[0] - require.Equal(t, "bear-mount", u.ID) + // #197: a merged-pak staleness row's identity is the SYNTHETIC + // merged-pak mod, not the contributing "bear-mount" mod - + // CheckMergedPakStaleness is profile-scoped, not per-mod. + require.Equal(t, "merged-pak", u.ID) require.True(t, u.RecompileNeeded) require.Equal(t, u.FromVersion, u.ToVersion, "a staleness row has no real version change") require.Equal(t, "(base pak updated)", u.VersionLabel()) } // TestCoreProviderActions_ApplyUpdate_Recompile_AppliesAndRedeploys proves -// ApplyUpdate dispatches a RecompileNeeded row to Service.ApplyRecompile: -// the retained source is recompiled and redeployed, provable via the -// on-disk (LinkCopy) deployed bytes. +// ApplyUpdate dispatches a RecompileNeeded row to Service.ApplyMergedPakRegen +// (checking RecompileNeeded BEFORE resolving a real InstalledMod - see +// coreProvider.ApplyUpdate's own doc comment for the defect this guards): +// the retained source is merged and deployed, provable via the on-disk +// (LinkCopy) deployed bytes. func TestCoreProviderActions_ApplyUpdate_Recompile_AppliesAndRedeploys(t *testing.T) { actions, compiler, deployedPath := newRecompileActionsFixture(t) @@ -154,5 +168,5 @@ func TestCoreProviderActions_ApplyUpdate_Recompile_AppliesAndRedeploys(t *testin data, err := os.ReadFile(deployedPath) require.NoError(t, err) - require.Equal(t, "retained-exmodz-bytes", string(data), "redeploy must reflect the freshly recompiled bytes") + require.Equal(t, "retained-exmodz-bytes", string(data), "redeploy must reflect the freshly merged bytes") } From a564538b785fc7727d24aa6e437d56b538dc0405 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 21:07:01 -0400 Subject: [PATCH 73/96] chore: remove dead #196 per-mod compile fingerprint machinery (#197) --- internal/core/service.go | 32 --------------- internal/storage/cache/cache.go | 61 ---------------------------- internal/storage/cache/cache_test.go | 54 ------------------------ 3 files changed, 147 deletions(-) diff --git a/internal/core/service.go b/internal/core/service.go index d43f6a9..7fdede8 100644 --- a/internal/core/service.go +++ b/internal/core/service.go @@ -998,14 +998,6 @@ func resolveBasePak(game *domain.Game) (string, error) { return candidate, nil } -// compiledFileName turns a downloaded source filename into the cached -// output's name: same base name, .pak extension, matching Icarus's "_P.pak" -// override convention. -func compiledFileName(sourceFileName string) string { - base := strings.TrimSuffix(sourceFileName, filepath.Ext(sourceFileName)) - return base + "_P.pak" -} - // basePakIndexHash opens basePakPath and returns its footer IndexHash // (#196) - cheap (footer + primary-index region only; unrealpak.Open never // reads a pak's actual file payloads), matching the base pak Compile itself @@ -1020,30 +1012,6 @@ func basePakIndexHash(basePakPath string) (string, error) { return r.IndexHash(), nil } -// stageCompileFingerprint stages fileID's #196 compile fingerprint into -// stagePath: the base pak's IndexHash (cache.MarkBaseIndexHash) and a copy -// of the original .exmodz (cache.RetainedSourceName), so a later staleness -// check can detect the base pak changing, and a later recompile can run -// offline. Both land in the SAME atomic commit as the compiled pak - see -// commitStagedCache/commitStagedCacheWithMarker - so a partial write here -// can never separate a compiled pak from its fingerprint or retained -// source. Called by both compile sites (download: DownloadModToCache; -// import: Importer.Import) after Compile succeeds, before the commit. -func stageCompileFingerprint(stagePath, fileID, basePakPath, sourceFilePath string) error { - indexHash, err := basePakIndexHash(basePakPath) - if err != nil { - return err - } - if err := cache.MarkBaseIndexHash(stagePath, fileID, indexHash); err != nil { - return err - } - retainedPath := filepath.Join(stagePath, cache.RetainedSourceName(fileID)) - if err := copyFileStreaming(sourceFilePath, retainedPath); err != nil { - return fmt.Errorf("retaining compile source: %w", err) - } - return nil -} - // GetGame retrieves a game by ID func (s *Service) GetGame(gameID string) (*domain.Game, error) { game, ok := s.games[gameID] diff --git a/internal/storage/cache/cache.go b/internal/storage/cache/cache.go index 83b64be..9812150 100644 --- a/internal/storage/cache/cache.go +++ b/internal/storage/cache/cache.go @@ -250,67 +250,6 @@ func (c *Cache) HasFileIDs(gameID, sourceID, modID, version string, fileIDs []st return true } -// baseIndexHashPrefix names a compiled file's base-pak fingerprint marker -// (#196, "the Friday problem"): the game's base data.pak footer IndexHash at -// the moment the file was compiled, so a later staleness check can detect -// the base pak changing underneath a compiled mod without re-hashing -// anything. Reserved (ReservedPrefix) so ListFiles/Size/deploy skip it like -// every other lmm bookkeeping entry. -const baseIndexHashPrefix = ReservedPrefix + "basehash-" - -// MarkBaseIndexHash records fileID's compiled-against base pak IndexHash -// (hex-encoded) into versionDir - written into the STAGING directory -// alongside the compile's own completion marker and retained source, just -// before the atomic commit that publishes all three together (mirrors -// MarkFileCompleteWithMembers; see internal/core's stageCompileFingerprint). -// An unverifiable fileID is skipped, matching writeFileMarker's contract. -func MarkBaseIndexHash(versionDir, fileID, indexHash string) error { - if !VerifiableFileID(fileID) { - return nil - } - if err := os.MkdirAll(versionDir, 0755); err != nil { - return fmt.Errorf("creating cache dir for base index marker: %w", err) - } - path := filepath.Join(versionDir, baseIndexHashPrefix+fileID) - if err := os.WriteFile(path, []byte(indexHash), 0644); err != nil { - return fmt.Errorf("writing base index marker: %w", err) - } - return nil -} - -// BaseIndexHashes reads every recorded base-pak fingerprint marker in the -// (gameID, sourceID, modID, version) cache entry, keyed by the compiled -// file's own fileID - mirroring FileManifests' directory-walk style so -// staleness detection (#196) works uniformly regardless of how a fileID was -// assigned (a download's real DownloadableFile.ID, or an import's synthetic -// one - see internal/core's stageCompileFingerprint). A directory with no -// compiled entries - including one with no markers at all - returns an -// empty map, never an error. -func (c *Cache) BaseIndexHashes(gameID, sourceID, modID, version string) (map[string]string, error) { - versionDir := c.ModPath(gameID, sourceID, modID, version) - entries, err := os.ReadDir(versionDir) - if err != nil { - if os.IsNotExist(err) { - return map[string]string{}, nil - } - return nil, fmt.Errorf("reading base index markers: %w", err) - } - - hashes := make(map[string]string) - for _, entry := range entries { - fileID, ok := strings.CutPrefix(entry.Name(), baseIndexHashPrefix) - if !ok || entry.IsDir() || !VerifiableFileID(fileID) { - continue - } - body, err := os.ReadFile(filepath.Join(versionDir, entry.Name())) - if err != nil { - return nil, fmt.Errorf("reading base index marker %s: %w", entry.Name(), err) - } - hashes[fileID] = string(body) - } - return hashes, nil -} - // retainedSourcePrefix names a compiled file's retained source archive // (#196): the original .exmodz kept beside the compiled pak so a later // recompile - the base pak changed, not the mod - can run offline instead diff --git a/internal/storage/cache/cache_test.go b/internal/storage/cache/cache_test.go index ddb1800..bef0ba6 100644 --- a/internal/storage/cache/cache_test.go +++ b/internal/storage/cache/cache_test.go @@ -476,60 +476,6 @@ func TestCache_MarkFileCompleteWithMembers_UnverifiableIDs(t *testing.T) { assert.Empty(t, entries, "an unverifiable file ID must not produce a marker") } -// TestCache_BaseIndexHashes_RoundTrip mirrors TestCache_FileManifests_RoundTrip -// for the #196 base-pak fingerprint marker: write two, read them back keyed -// by fileID, and confirm a mod with none reports an empty map. -func TestCache_BaseIndexHashes_RoundTrip(t *testing.T) { - c := cache.New(t.TempDir()) - versionDir := c.ModPath("g", "src", "mod", "1.0") - - require.NoError(t, cache.MarkBaseIndexHash(versionDir, "file-a", "aaaa1111")) - require.NoError(t, cache.MarkBaseIndexHash(versionDir, "file-b", "bbbb2222")) - - hashes, err := c.BaseIndexHashes("g", "src", "mod", "1.0") - require.NoError(t, err) - assert.Equal(t, map[string]string{"file-a": "aaaa1111", "file-b": "bbbb2222"}, hashes) - - none, err := c.BaseIndexHashes("g", "src", "other-mod", "1.0") - require.NoError(t, err) - assert.Empty(t, none, "a version dir with no base-index markers reports an empty map, not an error") -} - -// TestCache_BaseIndexHashes_ExcludedFromContentEnumerators pins that base -// index markers are reserved bookkeeping (ReservedPrefix), never mod -// content: they must never be deployed, sized, or counted, exactly like -// completion markers (TestCache_ManifestMarkersStayReservedAndComplete). -func TestCache_BaseIndexHashes_ExcludedFromContentEnumerators(t *testing.T) { - c := cache.New(t.TempDir()) - - require.NoError(t, c.Store("g", "src", "mod", "1.0", "Bear_Mount_P.pak", []byte("12345"))) - require.NoError(t, cache.MarkBaseIndexHash(c.ModPath("g", "src", "mod", "1.0"), "file-a", "aaaa1111")) - - files, err := c.ListFiles("g", "src", "mod", "1.0") - require.NoError(t, err) - assert.Equal(t, []string{"Bear_Mount_P.pak"}, files, "base index markers must never be listed as content") - - size, err := c.Size("g", "src", "mod", "1.0") - require.NoError(t, err) - assert.Equal(t, int64(5), size, "base index marker bytes must not count toward cache size") -} - -// TestCache_MarkBaseIndexHash_UnverifiableIDs mirrors -// TestCache_MarkFileCompleteWithMembers_UnverifiableIDs: an unverifiable -// file ID is skipped rather than producing a marker. -func TestCache_MarkBaseIndexHash_UnverifiableIDs(t *testing.T) { - c := cache.New(t.TempDir()) - modPath := c.ModPath("g", "src", "mod", "1.0") - require.NoError(t, os.MkdirAll(modPath, 0755)) - - require.NoError(t, cache.MarkBaseIndexHash(modPath, "", "aaaa1111")) - require.NoError(t, cache.MarkBaseIndexHash(modPath, "../escape", "aaaa1111")) - - entries, err := os.ReadDir(modPath) - require.NoError(t, err) - assert.Empty(t, entries, "an unverifiable file ID must not produce a base index marker") -} - // TestCache_RetainedSourceName_IsReservedAndExcludedFromContent pins that a // retained compile source (#196) written under RetainedSourceName is // reserved bookkeeping, not a deployment member - it must never be listed, From 9f4921413422b889719e50eb751e7d7d0c7ddd53 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 21:07:53 -0400 Subject: [PATCH 74/96] test: pin locked-mod-does-not-block-merge semantics (#197, coordinator-confirmed design decision) --- internal/core/merged_pak_locked_test.go | 88 +++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 internal/core/merged_pak_locked_test.go diff --git a/internal/core/merged_pak_locked_test.go b/internal/core/merged_pak_locked_test.go new file mode 100644 index 0000000..af63d32 --- /dev/null +++ b/internal/core/merged_pak_locked_test.go @@ -0,0 +1,88 @@ +package core_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestLockedMod_DiffStillParticipatesInMerge: a locked mod's retained +// exmodz contributes to the merge exactly like an unlocked one - locking +// pins THAT mod's own VERSION, it does not exclude its diff or freeze the +// merged pak (design decision 3, coordinator-confirmed). +func TestLockedMod_DiffStillParticipatesInMerge(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + pm := svc.NewProfileManager() + require.NoError(t, pm.SetModLock(game.ID, "default", "fake-compiler", "bear-mount", "")) + + warnings, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err, "a locked mod must not block the merge") + require.Empty(t, warnings) + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + data, err := os.ReadFile(deployedPath) + require.NoError(t, err) + require.Equal(t, "bear-bytes", string(data), "the locked mod's own diff must be included") +} + +// TestLockedMod_DoesNotBlockAnotherModsChangeFromReachingTheMerge: enabling +// a SECOND, unlocked mod alongside a locked one must still trigger +// regeneration and include BOTH mods' diffs. +func TestLockedMod_DoesNotBlockAnotherModsChangeFromReachingTheMerge(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + pm := svc.NewProfileManager() + require.NoError(t, pm.SetModLock(game.ID, "default", "fake-compiler", "bear-mount", "")) + _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "wolf-mount", "1.0", "exmodz-file", []byte("wolf-bytes")) + + warnings, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err, "a lock on one mod must never block ANOTHER mod's change from reaching the merged pak") + require.Empty(t, warnings) + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + data, err := os.ReadFile(deployedPath) + require.NoError(t, err) + require.Equal(t, "bear-byteswolf-bytes", string(data), "both mods' diffs must be present - the lock excluded neither") +} + +// TestLockedMod_CheckMergedPakStaleness_NotBlockedByLock proves the CHECK +// side (not just apply) also treats a locked mod normally. +func TestLockedMod_CheckMergedPakStaleness_NotBlockedByLock(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + pm := svc.NewProfileManager() + require.NoError(t, pm.SetModLock(game.ID, "default", "fake-compiler", "bear-mount", "")) + + upd, err := svc.CheckMergedPakStaleness(game, "default") + require.NoError(t, err) + require.NotNil(t, upd, "a never-yet-generated merged pak is stale regardless of a lock elsewhere in the profile") + + _, err = svc.ApplyMergedPakRegen(context.Background(), game, "default", nil) + require.NoError(t, err) + + upd, err = svc.CheckMergedPakStaleness(game, "default") + require.NoError(t, err) + require.Nil(t, upd, "after applying, the locked mod's presence must not cause a spurious permanent-stale state") +} + +// TestLockedMod_ApplyMergedPakRegen_NeverErrorsForALock proves +// ApplyMergedPakRegen has NO lock-gate at all (unlike #196's ApplyRecompile, +// which refused a locked MOD's own recompile) - it is a profile-level +// operation, and design decision 3 explicitly rejects "freeze the whole +// merge on any lock present." +func TestLockedMod_ApplyMergedPakRegen_NeverErrorsForALock(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + pm := svc.NewProfileManager() + require.NoError(t, pm.SetModLock(game.ID, "default", "fake-compiler", "bear-mount", "")) + + _, err := svc.ApplyMergedPakRegen(context.Background(), game, "default", nil) + require.NoError(t, err, "ApplyMergedPakRegen must never refuse due to a lock - #196's ErrModLocked gate does not apply here") +} From b8ae4e724c4da47653ac2f02b40b14d810accf2b Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 21:09:02 -0400 Subject: [PATCH 75/96] docs: amend #196 CHANGELOG entry for merged-pak compilation (#197) --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2f31b2..aacff23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Icarus built-in mod source** (`internal/source/icarus`): a public, unauthenticated Firestore-backed catalog (Project Daedalus) — `lmm search`/`install`/`update` work against it like NexusMods/CurseForge. A `.exmodz` mod file now compiles into a deployable `_P.pak` at download time via a new, game-agnostic `internal/unrealpak` PAK reader/writer and the new `deploy_mode: compile` game setting; a plain `.pak` file from the same catalog is unaffected and deploys through the existing extract/copy pipeline unchanged. Base data tables are read directly from the installed game's own `data.pak`, so a compile always matches the installed game version and works entirely offline; `internal/unrealpak` reads both the stored and the Zlib-compressed entries that pak contains, using only the standard library (#136, #175) - `lmm game detect` now recognizes Icarus (Steam App ID `1149460`) and generates a complete `games.yaml` entry for it (`deploy_mode: compile`, `sources: {icarus: icarus}`) — no more hand-editing `games.yaml` to get started. The known-games schema (`steam-games.yaml`, built-in or your own override) gained two optional fields, `deploy_mode` and `sources`, generalizing detection beyond NexusMods-only games; every existing entry is unaffected (#177) - Custom `api` sources' `search` endpoint gains `{category}`/`{tags}` path placeholders, fed from `SearchQuery.Category`/`.Tags` (URL-escaped; multiple tags comma-joined) — previously these were silently dropped with no way for a declarative source to express category/tag filtering. A definition whose `search` path omits the new placeholders is unaffected: the values are computed but never substituted in, matching today's behavior exactly (#120) -- Compiled mods (`deploy_mode: compile`, e.g. Icarus) now recover automatically when the game's base `data.pak` changes underneath them — "the Friday problem": a weekly base-pak refresh used to silently revert a compiled mod's patched tables, with nothing to notice. Compiling now records the base pak's footer fingerprint and retains a copy of the original `.exmodz` beside the compiled `_P.pak`, both invisible to deployment. `lmm update` (CLI and TUI) checks every compiled mod's fingerprint against the game's current base pak and reports a same-version "recompile needed" row (additive `--json` field `recompile_needed`/`reason`) alongside normal version updates; applying it recompiles in place from the retained `.exmodz` (falling back to a re-download when possible) and redeploys — pinned mods recompile normally, locked mods are refused with the same loud lock warning a real update gets. Pre-existing compiled installs without the new fingerprint are left alone rather than guessed at (indistinguishable from a plain prebuilt `.pak`); they pick up fingerprinting on their next real recompile. `lmm verify` gains a matching "RECOMPILE NEEDED" warning row (`stale_compile`) (#196) +- Compiled mods (`deploy_mode: compile`, e.g. Icarus) with more than one enabled `.exmodz` mod now compose correctly instead of silently shadowing each other: every enabled mod's table-row diffs are applied sequentially, in profile load order, into ONE merged `zzz_LMM_Merged_P.pak` per profile (named to mount last, so it always wins over a plain prebuilt `.pak`'s own table override) — two mods patching different fields of the same row, or entirely different rows of the same table, both survive; only a genuine same-field conflict is last-wins, and a bundled-asset path collision (which can't compose) is last-wins with a loud warning. This also fixes "the Friday problem" (a weekly base-pak refresh silently reverting a mod's patched tables, with nothing to notice): the merge regenerates whenever the enabled-mod set, load order, a mod's version, or the base pak itself changes. `lmm update` (CLI and TUI) reports a "recompile needed" row for the profile's merged pak (additive `--json` field `recompile_needed`/`reason`) alongside normal version updates; applying it regenerates and redeploys — pinned mods' diffs recompile normally, and a LOCKED mod's diff still participates in every merge (a lock pins that mod's own version, not the profile's merged pak). Installing/importing a `.exmodz` now only validates and retains it (a per-mod compiled pak is no longer generated or deployed); a plain prebuilt `.pak` mod, and every non-`deploy_mode: compile` game, is completely unaffected. `lmm verify` gains a matching "RECOMPILE NEEDED" warning row (`stale_compile`) for the profile's merged pak (#136, #175, #196, #197) ### Changed From 24f47a067bfecdd81d256fc13d7f2809c80cef81 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 21:27:51 -0400 Subject: [PATCH 76/96] fix: imported .exmodz mods now participate in merges (C1, #197 final review) Import retained its source keyed by the archive filename but never recorded that filename in the row's FileIDs (only a resolved source file ID, or nothing without --id) - enabledExmodzSources could never find it, so an imported mod silently never joined any merge, forever, and was invisible to update/verify on both sides of the staleness fingerprint. ImportResult now reports RetainedFileID (the identity Import actually retained under); cmd/lmm/import.go folds it into FileIDs and calls the newly-public Service.SyncMergedPak (promoted from the test-only SyncMergedPakForTest, since production callers now need it too) after saving the mod. --- cmd/lmm/import.go | 32 +++++++ cmd/lmm/import_compile_test.go | 97 ++++++++++++++++++++++ internal/core/importer.go | 14 ++++ internal/core/merged_pak.go | 11 ++- internal/core/merged_pak_hooks_test.go | 8 +- internal/core/merged_pak_locked_test.go | 6 +- internal/core/merged_pak_staleness_test.go | 6 +- internal/core/merged_pak_test.go | 22 ++--- 8 files changed, 171 insertions(+), 25 deletions(-) create mode 100644 cmd/lmm/import_compile_test.go diff --git a/cmd/lmm/import.go b/cmd/lmm/import.go index bf0a133..6a4d73b 100644 --- a/cmd/lmm/import.go +++ b/cmd/lmm/import.go @@ -200,6 +200,26 @@ func doImport(ctx context.Context, cmd *cobra.Command, service *core.Service, ga fmt.Fprintf(os.Stderr, "Warning: could not mark cache entry complete: %v\n", err) } } + // #197 C1 fix: a DeployCompile ".exmodz" import retains its source under + // RetainedFileID (the archive's own filename - Import's only stable + // identity), which is NEVER resolvedFile.ID (a real source file ID, or + // nothing at all without --id). Without folding it into FileIDs too, + // enabledExmodzSources can never find this mod's retained source - it + // silently never participates in any merge, forever, and is invisible + // to update/verify since it's excluded from both sides of the + // staleness fingerprint as well. + if result.RetainedFileID != "" { + found := false + for _, id := range importedFileIDs { + if id == result.RetainedFileID { + found = true + break + } + } + if !found { + importedFileIDs = append(importedFileIDs, result.RetainedFileID) + } + } // Show detection results fmt.Printf("\nMod: %s\n", result.Mod.Name) @@ -358,6 +378,18 @@ func doImport(ctx context.Context, cmd *cobra.Command, service *core.Service, ga } } + // #197 I3/C1 fix: a DeployCompile ".exmodz" import deploys zero files of + // its own (validate+retain only) - without this, the imported mod's + // content never reaches the game directory until some OTHER flow + // happens to sync the merged pak. + if syncWarnings, syncErr := service.SyncMergedPak(ctx, game, profileName); syncErr != nil { + fmt.Fprintf(os.Stderr, "Warning: could not sync merged pak: %v\n", syncErr) + } else { + for _, w := range syncWarnings { + fmt.Fprintf(os.Stderr, "Warning: %s\n", w) + } + } + // Run install.after_each hook if hookRunner != nil && resolvedHooks != nil && resolvedHooks.Install.AfterEach != "" { hookCtx.HookName = "install.after_each" diff --git a/cmd/lmm/import_compile_test.go b/cmd/lmm/import_compile_test.go new file mode 100644 index 0000000..d3bc80d --- /dev/null +++ b/cmd/lmm/import_compile_test.go @@ -0,0 +1,97 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/core" + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/spf13/cobra" + "github.com/stretchr/testify/require" +) + +// setupDoImportCompileTest builds a DeployCompile game with a registered +// merge-compiler-capable source and an installed base pak - the fixture +// `lmm import .exmodz` needs to exercise the real archive-mode +// ingest branch end to end. +func setupDoImportCompileTest(t *testing.T) (*core.Service, *domain.Game, *compilerInstallSource) { + t.Helper() + + configDir = t.TempDir() + dataDir = t.TempDir() + + installDir := t.TempDir() + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + writeFakeBasePak(t, basePak) + + svc, err := core.NewService(core.ServiceConfig{ConfigDir: configDir, DataDir: dataDir, CacheDir: t.TempDir()}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + compiler := &compilerInstallSource{fakeInstallSource: newFakeInstallSource("fake-compiler")} + svc.RegisterSource(compiler) + + game := &domain.Game{ + ID: "icarus", Name: "Icarus", InstallPath: installDir, ModPath: t.TempDir(), + DeployMode: domain.DeployCompile, LinkMethod: domain.LinkCopy, + SourceIDs: map[string]string{"fake-compiler": "external-icarus-id"}, + } + require.NoError(t, svc.AddGame(game)) + + oldProfile, oldSource, oldModID, oldForce, oldDryRun, oldSkipMatch := + importProfile, importSource, importModID, importForce, importDryRun, importSkipMatch + oldVerbose, oldNoColor, oldNoHooks := verbose, noColor, noHooks + importProfile = "" + importSource = "" + importModID = "" + importForce = true + importDryRun = false + importSkipMatch = true + verbose = false + noColor = true + noHooks = false + t.Cleanup(func() { + importProfile, importSource, importModID, importForce, importDryRun, importSkipMatch = + oldProfile, oldSource, oldModID, oldForce, oldDryRun, oldSkipMatch + verbose, noColor, noHooks = oldVerbose, oldNoColor, oldNoHooks + }) + + return svc, game, compiler +} + +// TestDoImport_DeployCompile_ImportedModParticipatesInMerge is the #197 C1 +// regression test: BEFORE the fix, an imported ".exmodz" mod's retained +// source was keyed by the archive's own filename +// (cache.RetainedSourceName(filename)) but the DB row's FileIDs never +// included that filename - only a resolved source file ID (with --id) or +// nothing at all (without --id, exercised here) - so enabledExmodzSources +// could never find it and the mod silently never participated in any +// merge, forever, while `lmm update`/`verify` reported everything healthy +// (the mod was equally invisible on both sides of the staleness +// fingerprint). This proves an imported mod's row-diff actually lands in +// the deployed merged pak. +func TestDoImport_DeployCompile_ImportedModParticipatesInMerge(t *testing.T) { + svc, game, _ := setupDoImportCompileTest(t) + + archivePath := filepath.Join(t.TempDir(), "Bear_Mount.exmodz") + require.NoError(t, os.WriteFile(archivePath, []byte("bear-exmodz-bytes"), 0o644)) + + _, err := captureStdoutErr(t, func() error { + return doImport(context.Background(), &cobra.Command{}, svc, game, []string{archivePath}) + }) + require.NoError(t, err) + + prof, err := svc.NewProfileManager().Get(game.ID, "default") + require.NoError(t, err) + require.Len(t, prof.Mods, 1) + require.Contains(t, prof.Mods[0].FileIDs, "Bear_Mount.exmodz", + "the row's FileIDs must include the archive filename - the ONLY identity the retained source is keyed by") + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + data, readErr := os.ReadFile(deployedPath) + require.NoError(t, readErr, "the imported mod's content must be deployed via the merged pak - import must sync it") + require.Equal(t, "bear-exmodz-bytes", string(data), "the merged pak must contain the imported mod's own retained content") +} diff --git a/internal/core/importer.go b/internal/core/importer.go index f6e2dca..36c7fe4 100644 --- a/internal/core/importer.go +++ b/internal/core/importer.go @@ -28,6 +28,17 @@ type ImportResult struct { FilesExtracted int LinkedSource string // "nexusmods", "local", etc. AutoDetected bool // true if source/ID was parsed from filename + // RetainedFileID is the identity Import used to key a retained compile + // source (cache.RetainedSourceName) - only set for the DeployCompile + // ".exmodz" branch, where it is the archive's own filename (Import has + // no other stable identity to retain under; see that branch's own doc + // comment). Callers MUST fold this into the InstalledMod/ModReference's + // FileIDs (#197 C1 fix): enabledExmodzSources walks FileIDs to find each + // mod's retained source, and a row whose FileIDs never includes this + // value is invisible to every future merge - silently, forever, since + // it is also excluded from the staleness fingerprint on both sides of + // the comparison. Empty for every other deploy mode. + RetainedFileID string } // Importer handles importing mods from local archive files @@ -105,6 +116,7 @@ func (i *Importer) Import(ctx context.Context, archivePath string, game *domain. var modName string var fileCount int + var retainedFileID string // Handle based on game's deploy mode if game.DeployMode == domain.DeployCompile && isExmodzFile(filename) { @@ -150,6 +162,7 @@ func (i *Importer) Import(ctx context.Context, archivePath string, game *domain. return nil, err } fileCount = 0 + retainedFileID = filename } else if game.DeployMode == domain.DeployCopy { // Copy mode: just copy the file as-is to cache (don't extract) modName = strings.TrimSuffix(filename, filepath.Ext(filename)) @@ -238,6 +251,7 @@ func (i *Importer) Import(ctx context.Context, archivePath string, game *domain. FilesExtracted: fileCount, LinkedSource: sourceID, AutoDetected: autoDetected, + RetainedFileID: retainedFileID, }, nil } diff --git a/internal/core/merged_pak.go b/internal/core/merged_pak.go index e102add..6b46854 100644 --- a/internal/core/merged_pak.go +++ b/internal/core/merged_pak.go @@ -240,10 +240,13 @@ func (s *Service) syncMergedPak(ctx context.Context, game *domain.Game, profileN return warnings, nil } -// SyncMergedPakForTest exposes syncMergedPak to external (core_test -// package) tests - see enabledExmodzSources/EnabledExmodzSourcesForTest's -// identical rationale. -func (s *Service) SyncMergedPakForTest(ctx context.Context, game *domain.Game, profileName string) ([]string, error) { +// SyncMergedPak exposes syncMergedPak as the public entry point every +// mutation flow that can change a merged pak's inputs (enabled-mod set, +// load order, mod version, base pak) must call after the mutation is +// durable (#197 fix wave: rollback, purge, and both import paths were +// found missing this call). Safe to call unconditionally - it no-ops for a +// non-DeployCompile game and is a cheap fast-path when nothing changed. +func (s *Service) SyncMergedPak(ctx context.Context, game *domain.Game, profileName string) ([]string, error) { return s.syncMergedPak(ctx, game, profileName) } diff --git a/internal/core/merged_pak_hooks_test.go b/internal/core/merged_pak_hooks_test.go index e67ab60..661b5b4 100644 --- a/internal/core/merged_pak_hooks_test.go +++ b/internal/core/merged_pak_hooks_test.go @@ -31,7 +31,7 @@ func TestEnableMod_SyncsMergedPak(t *testing.T) { func TestDisableMod_SyncsMergedPak_RemovesWhenLastModDisabled(t *testing.T) { svc, game, _ := newMergedPakTestGame(t) seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) - _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + _, err := svc.SyncMergedPak(context.Background(), game, "default") require.NoError(t, err) deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") _, err = os.Stat(deployedPath) @@ -49,7 +49,7 @@ func TestDisableMod_SyncsMergedPak_RemovesWhenLastModDisabled(t *testing.T) { func TestUninstallMod_SyncsMergedPak_RemovesWhenLastModUninstalled(t *testing.T) { svc, game, _ := newMergedPakTestGame(t) seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) - _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + _, err := svc.SyncMergedPak(context.Background(), game, "default") require.NoError(t, err) deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") @@ -110,7 +110,7 @@ func TestReorderProfileMods_SyncsMergedPak(t *testing.T) { svc, game, _ := newMergedPakTestGame(t) seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-a", []byte("A")) seedEnabledExmodzMod(t, svc, game, "fake-compiler", "wolf-mount", "1.0", "exmodz-b", []byte("B")) - _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + _, err := svc.SyncMergedPak(context.Background(), game, "default") require.NoError(t, err) deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") before, err := os.ReadFile(deployedPath) @@ -124,7 +124,7 @@ func TestReorderProfileMods_SyncsMergedPak(t *testing.T) { }) require.NoError(t, err) - _, err = svc.SyncMergedPakForTest(context.Background(), game, "default") + _, err = svc.SyncMergedPak(context.Background(), game, "default") require.NoError(t, err) after, err := os.ReadFile(deployedPath) require.NoError(t, err) diff --git a/internal/core/merged_pak_locked_test.go b/internal/core/merged_pak_locked_test.go index af63d32..1f5dccc 100644 --- a/internal/core/merged_pak_locked_test.go +++ b/internal/core/merged_pak_locked_test.go @@ -19,7 +19,7 @@ func TestLockedMod_DiffStillParticipatesInMerge(t *testing.T) { pm := svc.NewProfileManager() require.NoError(t, pm.SetModLock(game.ID, "default", "fake-compiler", "bear-mount", "")) - warnings, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + warnings, err := svc.SyncMergedPak(context.Background(), game, "default") require.NoError(t, err, "a locked mod must not block the merge") require.Empty(t, warnings) @@ -37,12 +37,12 @@ func TestLockedMod_DoesNotBlockAnotherModsChangeFromReachingTheMerge(t *testing. seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) pm := svc.NewProfileManager() require.NoError(t, pm.SetModLock(game.ID, "default", "fake-compiler", "bear-mount", "")) - _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + _, err := svc.SyncMergedPak(context.Background(), game, "default") require.NoError(t, err) seedEnabledExmodzMod(t, svc, game, "fake-compiler", "wolf-mount", "1.0", "exmodz-file", []byte("wolf-bytes")) - warnings, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + warnings, err := svc.SyncMergedPak(context.Background(), game, "default") require.NoError(t, err, "a lock on one mod must never block ANOTHER mod's change from reaching the merged pak") require.Empty(t, warnings) diff --git a/internal/core/merged_pak_staleness_test.go b/internal/core/merged_pak_staleness_test.go index 6a41e0f..58d1c30 100644 --- a/internal/core/merged_pak_staleness_test.go +++ b/internal/core/merged_pak_staleness_test.go @@ -13,7 +13,7 @@ import ( func TestCheckMergedPakStaleness_NotStaleWhenUnchanged(t *testing.T) { svc, game, _ := newMergedPakTestGame(t) seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) - _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + _, err := svc.SyncMergedPak(context.Background(), game, "default") require.NoError(t, err) upd, err := svc.CheckMergedPakStaleness(game, "default") @@ -24,7 +24,7 @@ func TestCheckMergedPakStaleness_NotStaleWhenUnchanged(t *testing.T) { func TestCheckMergedPakStaleness_StaleAfterModEnable(t *testing.T) { svc, game, _ := newMergedPakTestGame(t) seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) - _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + _, err := svc.SyncMergedPak(context.Background(), game, "default") require.NoError(t, err) seedEnabledExmodzMod(t, svc, game, "fake-compiler", "wolf-mount", "1.0", "exmodz-file", []byte("wolf-bytes")) @@ -57,7 +57,7 @@ func TestCheckMergedPakStaleness_NonCompileGame_Nil(t *testing.T) { func TestApplyMergedPakRegen_Regenerates(t *testing.T) { svc, game, _ := newMergedPakTestGame(t) seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) - _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + _, err := svc.SyncMergedPak(context.Background(), game, "default") require.NoError(t, err) seedEnabledExmodzMod(t, svc, game, "fake-compiler", "wolf-mount", "1.0", "exmodz-file", []byte("wolf-bytes")) diff --git a/internal/core/merged_pak_test.go b/internal/core/merged_pak_test.go index 089584d..1003afb 100644 --- a/internal/core/merged_pak_test.go +++ b/internal/core/merged_pak_test.go @@ -127,7 +127,7 @@ func TestSyncMergedPak_GeneratesAndDeploys(t *testing.T) { svc, game, _ := newMergedPakTestGame(t) seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-exmodz-bytes")) - warnings, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + warnings, err := svc.SyncMergedPak(context.Background(), game, "default") require.NoError(t, err) require.Empty(t, warnings) @@ -144,7 +144,7 @@ func TestSyncMergedPak_NoOpWhenUnchanged(t *testing.T) { svc, game, _ := newMergedPakTestGame(t) seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-exmodz-bytes")) - _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + _, err := svc.SyncMergedPak(context.Background(), game, "default") require.NoError(t, err) srcRaw, err := svc.GetSource("fake-compiler") @@ -153,7 +153,7 @@ func TestSyncMergedPak_NoOpWhenUnchanged(t *testing.T) { require.True(t, ok) require.Equal(t, 1, src.compileCalls) - _, err = svc.SyncMergedPakForTest(context.Background(), game, "default") + _, err = svc.SyncMergedPak(context.Background(), game, "default") require.NoError(t, err) require.Equal(t, 1, src.compileCalls, "an unchanged fingerprint must not trigger a second merge") } @@ -164,12 +164,12 @@ func TestSyncMergedPak_RegeneratesOnModEnable(t *testing.T) { svc, game, _ := newMergedPakTestGame(t) seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) - _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + _, err := svc.SyncMergedPak(context.Background(), game, "default") require.NoError(t, err) seedEnabledExmodzMod(t, svc, game, "fake-compiler", "wolf-mount", "1.0", "exmodz-file", []byte("wolf-bytes")) - warnings, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + warnings, err := svc.SyncMergedPak(context.Background(), game, "default") require.NoError(t, err) require.Empty(t, warnings) @@ -192,7 +192,7 @@ func TestSyncMergedPak_ZeroEnabledMods_UninstallsExistingPak(t *testing.T) { svc, game, _ := newMergedPakTestGame(t) seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) - _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + _, err := svc.SyncMergedPak(context.Background(), game, "default") require.NoError(t, err) deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") _, err = os.Stat(deployedPath) @@ -200,7 +200,7 @@ func TestSyncMergedPak_ZeroEnabledMods_UninstallsExistingPak(t *testing.T) { require.NoError(t, svc.SetModEnabled("fake-compiler", "bear-mount", game.ID, "default", false)) - _, err = svc.SyncMergedPakForTest(context.Background(), game, "default") + _, err = svc.SyncMergedPak(context.Background(), game, "default") require.NoError(t, err) _, err = os.Stat(deployedPath) @@ -214,13 +214,13 @@ func TestSyncMergedPak_RegeneratesOnBaseHashChange(t *testing.T) { svc, game, basePak := newMergedPakTestGame(t) seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) - _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + _, err := svc.SyncMergedPak(context.Background(), game, "default") require.NoError(t, err) // Rewrite the base pak with different content - a new IndexHash. writeFakeBasePakWithTable(t, basePak, map[string][]byte{"AI/D_Other.json": []byte(`{"Rows":[{"Name":"x","V":1}]}`)}) - _, err = svc.SyncMergedPakForTest(context.Background(), game, "default") + _, err = svc.SyncMergedPak(context.Background(), game, "default") require.NoError(t, err) srcRaw, err := svc.GetSource("fake-compiler") @@ -238,7 +238,7 @@ func TestSyncMergedPak_NonCompileGame_NoOp(t *testing.T) { game := &domain.Game{ID: "skyrim-se", ModPath: t.TempDir(), DeployMode: domain.DeployExtract} require.NoError(t, svc.AddGame(game)) - warnings, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + warnings, err := svc.SyncMergedPak(context.Background(), game, "default") require.NoError(t, err) require.Empty(t, warnings) } @@ -254,7 +254,7 @@ func TestSyncMergedPak_AssetCollisionWarningSurfaces(t *testing.T) { src.mergeWarnings = []string{"asset collision: fixture warning"} seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) - warnings, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + warnings, err := svc.SyncMergedPak(context.Background(), game, "default") require.NoError(t, err) require.Equal(t, []string{"asset collision: fixture warning"}, warnings) } From be11ce80bb2687ddd64e5f329318d787ec80860e Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 21:32:14 -0400 Subject: [PATCH 77/96] fix: sync merged pak on rollback; purge undeploys it explicitly (I1/I2, #197 final review) I1: ApplyRollback moves a mod's Version (and FileIDs) - both regeneration triggers - without ever syncing the merged pak, so a rolled-back mod's stale diff stayed deployed until an unrelated flow happened to sync it. I2: lmm purge's own contract is 'remove ALL deployed mod files, resetting the game directory to its pre-modded state', but exmodz mods deploy exclusively through the shared merged pak (zero per-mod deployment members), so purgeMods' per-mod Installer.Uninstall loop was a no-op for every one of them - a plain purge also can't rely on syncMergedPak's fingerprint diff, since purge intentionally leaves Enabled untouched (only Deployed flips), so the merge inputs never change. New Service.PurgeMergedPak explicitly undeploys it, mirroring purge's own --uninstall/keep-cache distinction for real mods. --- internal/core/flows.go | 24 ++++++- internal/core/merged_pak.go | 40 +++++++++++ internal/core/merged_pak_hooks_test.go | 92 ++++++++++++++++++++++++++ 3 files changed, 155 insertions(+), 1 deletion(-) diff --git a/internal/core/flows.go b/internal/core/flows.go index 3da1166..be4b204 100644 --- a/internal/core/flows.go +++ b/internal/core/flows.go @@ -2239,7 +2239,17 @@ func (s *Service) PurgeProfile(ctx context.Context, game *domain.Game, profileNa skipped: &result.Skipped, purged: &result.Purged, }) - return result, err + if err != nil { + return result, err + } + + // #197 I2 fix: see PurgeMergedPak's own doc comment - exmodz mods have + // no per-mod deployment for the loop above to have already undeployed. + if perr := s.PurgeMergedPak(ctx, game, profileName, opts.Uninstall); perr != nil { + result.Notes = append(result.Notes, fmt.Sprintf("Warning: could not remove merged pak: %v", perr)) + } + + return result, nil } // SwitchPlan is the pure, displayable diff between the currently-active @@ -4807,6 +4817,18 @@ func (s *Service) ApplyRollback(ctx context.Context, game *domain.Game, profileN return result, fmt.Errorf("updating profile: %w", err) } + // #197 I1 fix: a rollback changes the mod's Version (and possibly its + // FileIDs), both regeneration triggers - without this, the merged pak + // keeps the rolled-away-from version's diff until some OTHER flow + // happens to sync it. + if syncWarnings, syncErr := s.syncMergedPak(ctx, game, profileName); syncErr != nil { + result.Notes = append(result.Notes, fmt.Sprintf("Warning: could not sync merged pak: %v", syncErr)) + } else { + for _, w := range syncWarnings { + result.Notes = append(result.Notes, "Warning: "+w) + } + } + return result, nil } diff --git a/internal/core/merged_pak.go b/internal/core/merged_pak.go index 6b46854..1c2f342 100644 --- a/internal/core/merged_pak.go +++ b/internal/core/merged_pak.go @@ -376,3 +376,43 @@ func (s *Service) ApplyMergedPakRegen(ctx context.Context, game *domain.Game, pr } return result, nil } + +// PurgeMergedPak explicitly undeploys game+profileName's merged pak (#197 +// I2 fix). `lmm purge`'s own contract is "remove ALL deployed mod files... +// resetting the game directory back to its pre-modded state" - but exmodz +// mods deploy EXCLUSIVELY through this one shared artifact, never their +// own per-mod cache entry (Task 2/3), so purgeMods' per-real-mod +// Installer.Uninstall loop is a no-op for every one of them. Nor can this +// be left to syncMergedPak's own fingerprint-diffing: a plain (non- +// --uninstall) purge intentionally leaves each mod's Enabled bit +// untouched (only Deployed flips), so the merge INPUTS never change and +// syncMergedPak's fast path would silently do nothing, leaving the pak +// deployed - the exact regression this fixes. +// +// deleteCache mirrors purge's own --uninstall distinction for real mods: +// false (plain purge) undeploys the FILE but keeps the cache entry and +// fingerprint, so a later `lmm deploy` redeploys the identical merged pak +// without recomputing anything (relies on syncMergedPak's fast path also +// confirming the deployed artifact still exists - #197 I5's fix); true +// (--uninstall) also clears the cache entry, matching every real mod's +// full removal. +func (s *Service) PurgeMergedPak(ctx context.Context, game *domain.Game, profileName string, deleteCache bool) error { + if game.DeployMode != domain.DeployCompile { + return nil + } + installer, err := s.GetInstallerForProfile(game, profileName) + if err != nil { + return err + } + syntheticMod := &domain.Mod{ID: mergedPakModID, SourceID: domain.SourceMerged, Version: mergedPakVersion, GameID: game.ID} + if err := installer.Uninstall(ctx, game, syntheticMod, profileName); err != nil { + return fmt.Errorf("removing merged pak: %w", err) + } + if deleteCache { + gameCache := s.GetGameCache(game) + if err := gameCache.Delete(game.ID, domain.SourceMerged, mergedPakModID, mergedPakVersion); err != nil { + return fmt.Errorf("clearing merged pak cache entry: %w", err) + } + } + return nil +} diff --git a/internal/core/merged_pak_hooks_test.go b/internal/core/merged_pak_hooks_test.go index 661b5b4..478f08f 100644 --- a/internal/core/merged_pak_hooks_test.go +++ b/internal/core/merged_pak_hooks_test.go @@ -8,6 +8,7 @@ import ( "github.com/DonovanMods/linux-mod-manager/internal/core" "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" "github.com/stretchr/testify/require" ) @@ -77,6 +78,97 @@ func TestDeployProfile_SyncsMergedPak(t *testing.T) { require.Equal(t, "bear-bytes", string(data)) } +// TestApplyRollback_SyncsMergedPak proves a rollback (a version + FileIDs +// change - a documented regeneration trigger) reaches the merged pak +// (#197 I1 fix - previously missed, so a rolled-back mod's stale diff +// stayed deployed until an unrelated flow happened to sync it). +func TestApplyRollback_SyncsMergedPak(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + + gameCache := svc.GetGameCache(game) + require.NoError(t, gameCache.Store(game.ID, "fake-compiler", "bear-mount", "1.0", cache.RetainedSourceName("exmodz-v1"), []byte("v1-bytes"))) + require.NoError(t, gameCache.Store(game.ID, "fake-compiler", "bear-mount", "2.0", cache.RetainedSourceName("exmodz-v2"), []byte("v2-bytes"))) + + require.NoError(t, svc.SaveInstalledMod(&domain.InstalledMod{ + Mod: domain.Mod{ID: "bear-mount", SourceID: "fake-compiler", Name: "bear-mount", Version: "2.0", GameID: game.ID}, + ProfileName: "default", + Enabled: true, + FileIDs: []string{"exmodz-v2"}, + PreviousVersion: "1.0", + PreviousFileIDs: []string{"exmodz-v1"}, + UpdatePolicy: domain.UpdateNotify, + })) + pm := svc.NewProfileManager() + require.NoError(t, pm.UpsertMod(game.ID, "default", domain.ModReference{SourceID: "fake-compiler", ModID: "bear-mount", Version: "2.0", FileIDs: []string{"exmodz-v2"}})) + + _, err := svc.SyncMergedPak(context.Background(), game, "default") + require.NoError(t, err) + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + before, err := os.ReadFile(deployedPath) + require.NoError(t, err) + require.Equal(t, "v2-bytes", string(before)) + + _, err = svc.ApplyRollback(context.Background(), game, "default", "fake-compiler", "bear-mount", core.RollbackOptions{}, nil) + require.NoError(t, err) + + after, err := os.ReadFile(deployedPath) + require.NoError(t, err) + require.Equal(t, "v1-bytes", string(after), "ApplyRollback must sync the merged pak to reflect the rolled-back version") +} + +// TestPurgeProfile_UndeploysMergedPak_KeepsCacheWithoutUninstall proves a +// plain `lmm purge` (no --uninstall) removes the deployed merged pak even +// though the purge loop's own per-mod Installer.Uninstall calls are no-ops +// for exmodz mods (zero deployment members of their own, #197 I2 fix) - +// and that the cache entry survives, matching purge's "keep records, +// redeploy later" contract for real mods. +func TestPurgeProfile_UndeploysMergedPak_KeepsCacheWithoutUninstall(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + _, err := svc.SyncMergedPak(context.Background(), game, "default") + require.NoError(t, err) + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + _, err = os.Stat(deployedPath) + require.NoError(t, err, "precondition: the merged pak must exist before purging") + + mod, err := svc.GetInstalledMod("fake-compiler", "bear-mount", game.ID, "default") + require.NoError(t, err) + + _, err = svc.PurgeProfile(context.Background(), game, "default", []domain.InstalledMod{*mod}, core.PurgeOptions{}, nil) + require.NoError(t, err) + + _, err = os.Stat(deployedPath) + require.True(t, os.IsNotExist(err), "purge must undeploy the merged pak, not just no-op per-mod") + + gameCache := svc.GetGameCache(game) + require.True(t, gameCache.Exists(game.ID, domain.SourceMerged, "merged-pak", "merged"), + "a plain purge (no --uninstall) must keep the merged pak's cache entry, mirroring real-mod purge semantics") +} + +// TestPurgeProfile_Uninstall_DeletesMergedPakCacheToo proves `lmm purge +// --uninstall` also clears the merged pak's cache entry, matching every +// real mod's full removal. +func TestPurgeProfile_Uninstall_DeletesMergedPakCacheToo(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + _, err := svc.SyncMergedPak(context.Background(), game, "default") + require.NoError(t, err) + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + + mod, err := svc.GetInstalledMod("fake-compiler", "bear-mount", game.ID, "default") + require.NoError(t, err) + + _, err = svc.PurgeProfile(context.Background(), game, "default", []domain.InstalledMod{*mod}, core.PurgeOptions{Uninstall: true}, nil) + require.NoError(t, err) + + _, err = os.Stat(deployedPath) + require.True(t, os.IsNotExist(err)) + + gameCache := svc.GetGameCache(game) + require.False(t, gameCache.Exists(game.ID, domain.SourceMerged, "merged-pak", "merged"), + "--uninstall must also clear the merged pak's cache entry") +} + // TestApplyProfileSwitch_SyncsMergedPakForToProfile proves switching TO a // profile with enabled exmodz mods deploys ITS merged pak (plan.To, not // plan.From). From e9d31b17f1ac15ceba0238ac858b1e358652ffdf Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 21:35:54 -0400 Subject: [PATCH 78/96] fix: sync merged pak on profile import and scan-mode import (I3, #197 final review) ApplyImport (profile import) and cmd/lmm/import.go's scan-mode tail both download/deploy/save/upsert mods with no merged-pak sync - since #197's ingest yields zero per-mod deployables, an imported profile's exmodz mods put no content in the game at all until some other flow happened to sync it. --- cmd/lmm/import.go | 12 ++ internal/core/flows.go | 16 +++ internal/core/merged_pak_import_flow_test.go | 128 +++++++++++++++++++ 3 files changed, 156 insertions(+) create mode 100644 internal/core/merged_pak_import_flow_test.go diff --git a/cmd/lmm/import.go b/cmd/lmm/import.go index 6a4d73b..b537b6b 100644 --- a/cmd/lmm/import.go +++ b/cmd/lmm/import.go @@ -778,6 +778,18 @@ func importExistingMod(ctx context.Context, service *core.Service, game *domain. } } + // #197 I3 fix: mirrors doImport's archive-mode tail - a scanned mod is a + // mod-set change for whatever profile it's registered into. + if syncWarnings, syncErr := service.SyncMergedPak(ctx, game, profileName); syncErr != nil { + if verbose { + fmt.Printf(" Warning: could not sync merged pak: %v\n", syncErr) + } + } else { + for _, w := range syncWarnings { + fmt.Fprintf(os.Stderr, "Warning: %s\n", w) + } + } + return nil } diff --git a/internal/core/flows.go b/internal/core/flows.go index be4b204..4e6d45a 100644 --- a/internal/core/flows.go +++ b/internal/core/flows.go @@ -5236,5 +5236,21 @@ func (s *Service) ApplyImport(ctx context.Context, game *domain.Game, plan *Impo emit(installedEvt) } + // #197 I3 fix: profile import deploys mods (installer.Install above) the + // same way ApplyInstall/DeployProfile do - without this, an imported + // profile's exmodz mods (zero per-mod deployment members of their own, + // Task 2/3) put NO content in the game directory at all until some + // OTHER flow happens to sync the merged pak. + if syncWarnings, syncErr := s.syncMergedPak(ctx, game, profile.Name); syncErr != nil { + msg := fmt.Sprintf("syncing merged pak: %v", syncErr) + result.Warnings = append(result.Warnings, msg) + emit(DeployProgress{Phase: ImportNote, Detail: msg}) + } else { + for _, w := range syncWarnings { + result.Warnings = append(result.Warnings, w) + emit(DeployProgress{Phase: ImportNote, Detail: w}) + } + } + return result, nil } diff --git a/internal/core/merged_pak_import_flow_test.go b/internal/core/merged_pak_import_flow_test.go new file mode 100644 index 0000000..32c4c78 --- /dev/null +++ b/internal/core/merged_pak_import_flow_test.go @@ -0,0 +1,128 @@ +package core_test + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/core" + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/source" + "github.com/DonovanMods/linux-mod-manager/internal/storage/config" + "github.com/stretchr/testify/require" +) + +// importFlowCompilerSource is a full ModSource + source.MergeCompiler fake +// serving one real HTTP download (an ".exmodz" file) - unlike +// fakeCompilerSource (service_icarus_compile_test.go), which stubs +// GetMod/GetModFiles as source.ErrNotSupported, ApplyImport's download +// loop needs a working GetMod->GetModFiles->DownloadMod chain end to end. +type importFlowCompilerSource struct { + mod *domain.Mod + fileName string + server *httptest.Server + content []byte +} + +func newImportFlowCompilerSource(mod *domain.Mod, fileName string, content []byte) *importFlowCompilerSource { + s := &importFlowCompilerSource{mod: mod, fileName: fileName, content: content} + s.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(s.content) + })) + return s +} + +func (s *importFlowCompilerSource) Close() { s.server.Close() } + +func (s *importFlowCompilerSource) ID() string { return s.mod.SourceID } +func (s *importFlowCompilerSource) Name() string { return "Import Flow Compiler Source" } +func (s *importFlowCompilerSource) AuthURL() string { return "" } +func (s *importFlowCompilerSource) ExchangeToken(ctx context.Context, code string) (*source.Token, error) { + return nil, source.ErrNotSupported +} +func (s *importFlowCompilerSource) Search(ctx context.Context, query source.SearchQuery) (source.SearchResult, error) { + return source.SearchResult{}, source.ErrNotSupported +} +func (s *importFlowCompilerSource) GetMod(ctx context.Context, gameID, modID string) (*domain.Mod, error) { + if modID == s.mod.ID { + return s.mod, nil + } + return nil, domain.ErrModNotFound +} +func (s *importFlowCompilerSource) GetDependencies(ctx context.Context, mod *domain.Mod) ([]domain.ModReference, error) { + return nil, nil +} +func (s *importFlowCompilerSource) GetModFiles(ctx context.Context, mod *domain.Mod) ([]domain.DownloadableFile, error) { + return []domain.DownloadableFile{{ID: "exmodz-1", Name: "Main", FileName: s.fileName, IsPrimary: true}}, nil +} +func (s *importFlowCompilerSource) GetDownloadURL(ctx context.Context, mod *domain.Mod, fileID string) (string, error) { + return s.server.URL, nil +} +func (s *importFlowCompilerSource) CheckUpdates(ctx context.Context, installed []domain.InstalledMod) ([]domain.Update, error) { + return nil, nil +} +func (s *importFlowCompilerSource) ValidateSource(sourceFilePath string) error { + _, err := os.Stat(sourceFilePath) + return err +} +func (s *importFlowCompilerSource) MergeCompile(ctx context.Context, basePakPath string, sources []source.MergeSource, outputPath string) ([]string, error) { + var out []byte + for _, src := range sources { + data, err := os.ReadFile(src.ExmodzPath) + if err != nil { + return nil, err + } + out = append(out, data...) + } + return nil, os.WriteFile(outputPath, out, 0o644) +} + +var ( + _ source.ModSource = (*importFlowCompilerSource)(nil) + _ source.MergeCompiler = (*importFlowCompilerSource)(nil) +) + +// TestApplyImport_DeployCompile_SyncsMergedPak is the #197 I3 regression +// test: importing a profile that references an ".exmodz" mod downloads +// and validates+retains it (Task 2/3 - zero per-mod deployment members), +// but nothing deployed it into the game directory until this fix, since +// ApplyImport never called syncMergedPak. +func TestApplyImport_DeployCompile_SyncsMergedPak(t *testing.T) { + svc := newFlowsTestService(t) + installDir := t.TempDir() + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + writeFakeBasePak(t, basePak) + + game := &domain.Game{ + ID: "icarus", Name: "Icarus", InstallPath: installDir, ModPath: t.TempDir(), + DeployMode: domain.DeployCompile, LinkMethod: domain.LinkCopy, + SourceIDs: map[string]string{"fake-compiler": "external-icarus-id"}, + } + require.NoError(t, svc.AddGame(game)) + + mod := &domain.Mod{ID: "bear-mount", SourceID: "fake-compiler", Name: "Bear Mount", Version: "1.0", GameID: "icarus"} + src := newImportFlowCompilerSource(mod, "Bear_Mount.exmodz", []byte("bear-exmodz-bytes")) + defer src.Close() + svc.RegisterSource(src) + + profile := &domain.Profile{Name: "target", GameID: game.ID, Mods: []domain.ModReference{{SourceID: "fake-compiler", ModID: "bear-mount", Version: "1.0"}}} + data, err := config.ExportProfile(profile) + require.NoError(t, err) + + plan, err := svc.PlanImport(context.Background(), game, data) + require.NoError(t, err) + require.Len(t, plan.Missing, 1) + + result, err := svc.ApplyImport(context.Background(), game, plan, core.ProfileImportOptions{}, nil) + require.NoError(t, err) + require.Equal(t, 1, result.Installed) + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + deployedData, err := os.ReadFile(deployedPath) + require.NoError(t, err, "ApplyImport must sync the merged pak - the imported mod deploys zero files of its own") + require.Equal(t, "bear-exmodz-bytes", string(deployedData)) +} From 60c14008126c8e40fc04c680f5147990c54dab1e Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 21:38:08 -0400 Subject: [PATCH 79/96] fix: verify no longer reports FILE COUNT MISMATCH for healthy exmodz mods (I4, #197 final review) The pre-pass compared checksum-row count (>=1 per downloaded exmodz) against cache.ListFiles - which excludes reserved entries and is now always 0 for a DeployCompile mod's validate+retain-only cache entry (Task 2/3, zero deployment members by design; the shared merged pak is what actually deploys). Every healthy Icarus profile therefore hit a permanent false positive. Fixed by skipping the check when the mod has a retained compile source on disk for any of its FileIDs - the one signal that distinguishes 'deploys nothing on purpose' from a genuinely corrupted cache entry, which the check still catches everywhere else (including a compile-mode game's own plain prebuilt .pak mods). --- cmd/lmm/verify.go | 30 +++++++++++++++++++++++ cmd/lmm/verify_recompile_test.go | 41 ++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/cmd/lmm/verify.go b/cmd/lmm/verify.go index 271002a..3d47dd0 100644 --- a/cmd/lmm/verify.go +++ b/cmd/lmm/verify.go @@ -11,6 +11,7 @@ import ( "github.com/DonovanMods/linux-mod-manager/internal/core" "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" "github.com/DonovanMods/linux-mod-manager/internal/storage/config" "github.com/spf13/cobra" @@ -162,6 +163,21 @@ func runVerify(cmd *cobra.Command, args []string) error { }) } +// hasRetainedSource reports whether any of fileIDs has a retained compile +// source (cache.RetainedSourceName) on disk for sourceID/modID/version - +// the signal that a cache entry is a DeployCompile ".exmodz" validate+ +// retain-only entry (#197 I4), which deploys zero files of its own by +// design and must not be flagged as a FILE COUNT MISMATCH. +func hasRetainedSource(gameCache *cache.Cache, gameID, sourceID, modID, version string, fileIDs []string) bool { + for _, fileID := range fileIDs { + retainedPath := gameCache.GetFilePath(gameID, sourceID, modID, version, cache.RetainedSourceName(fileID)) + if _, err := os.Stat(retainedPath); err == nil { + return true + } + } + return false +} + func doVerify(cmd *cobra.Command, svc *core.Service, game *domain.Game, args []string) error { profile, err := resolveProfile(svc, game.ID, verifyProfile) if err != nil { @@ -247,6 +263,20 @@ func doVerify(cmd *cobra.Command, svc *core.Service, game *domain.Game, args []s if modFilter != "" && mod.ID != modFilter { continue } + // #197 I4 fix: a DeployCompile game's ".exmodz" mod is ingested as + // validate+retain ONLY (Task 2/3) - it has zero deployment members + // of its own by design (the shared merged pak, checked separately + // above, is what actually deploys), so ListFiles == 0 here is + // correct, healthy state, not a mismatch. Detected by checking + // whether any of the mod's own FileIDs has a retained source on + // disk - that is the one signal that distinguishes "this entry + // deploys nothing on purpose" from a genuinely corrupted/emptied + // cache entry (which the check below must still catch for every + // OTHER deploy mode, and even for a compile-mode game's own plain + // prebuilt ".pak" mods, which still deploy normally). + if game.DeployMode == domain.DeployCompile && hasRetainedSource(gameCache, game.ID, mod.SourceID, mod.ID, mod.Version, mod.FileIDs) { + continue + } cacheExists := gameCache.Exists(game.ID, mod.SourceID, mod.ID, mod.Version) if !cacheExists { continue diff --git a/cmd/lmm/verify_recompile_test.go b/cmd/lmm/verify_recompile_test.go index f4082fe..5444a61 100644 --- a/cmd/lmm/verify_recompile_test.go +++ b/cmd/lmm/verify_recompile_test.go @@ -78,3 +78,44 @@ func TestDoVerify_StaleCompile_JSON(t *testing.T) { assert.Equal(t, "merged-pak", found.ModID) assert.GreaterOrEqual(t, out.Warnings, 1) } + +// TestDoVerify_HealthyExmodzMod_NoFileCountMismatch is the #197 I4 +// regression test: a DeployCompile ".exmodz" mod is ingested as +// validate+retain ONLY (Task 2/3) - it has zero deployment members of its +// own by design, so the pre-#197-I4 file-count pre-pass (checksum-row +// count vs cache.ListFiles) reported a false "FILE COUNT MISMATCH" for +// EVERY healthy exmodz mod, unconditionally. This proves a healthy, +// up-to-date exmodz mod verifies with no file_count_mismatch row at all. +func TestDoVerify_HealthyExmodzMod_NoFileCountMismatch(t *testing.T) { + svc, game, _, _ := setupDoUpdateRecompileTest(t) + require.NoError(t, svc.SaveFileChecksum("fake-compiler", "bear-mount", game.ID, "default", "exmodz-file-id", "deadbeef")) + // Sync so the merged pak is up to date - isolates the file-count check + // from the (separately tested) stale_compile row. + _, err := svc.SyncMergedPak(context.Background(), game, "default") + require.NoError(t, err) + + verifyProfile = "default" + jsonOutput = true + t.Cleanup(func() { verifyProfile = ""; jsonOutput = false }) + + cmd := &cobra.Command{} + cmd.SetContext(context.Background()) + + var buf bytes.Buffer + oldStdout := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + err = doVerify(cmd, svc, game, nil) + _ = w.Close() + os.Stdout = oldStdout + require.NoError(t, err) + _, _ = buf.ReadFrom(r) + + var out verifyJSONOutput + require.NoError(t, json.Unmarshal(buf.Bytes(), &out)) + + for _, f := range out.Files { + assert.NotEqual(t, "file_count_mismatch", f.Status, + "a healthy exmodz mod (validate+retain only, zero deployment members by design) must never report file_count_mismatch") + } +} From 789302641c8191db48e08a0d4788ee05c78afc13 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 21:40:49 -0400 Subject: [PATCH 80/96] fix: detect and self-heal a merged pak whose fingerprint matches but isn't deployed (I5, #197 final review) syncMergedPak committed the fingerprint to the cache entry BEFORE calling installer.Install - a failed Install (or the file vanishing some other way, e.g. a purge that intentionally keeps the cache entry, #197 I2) left every later call comparing stored==current and fast-pathing 'nothing changed', permanently wedged, with the game directory not actually holding the merged pak. CheckMergedPakStaleness (what lmm update/verify actually call) had the identical gap, so the safety net couldn't see it either. Both now confirm the deployed artifact is physically present before trusting a fingerprint match; syncMergedPak redeploys the EXISTING cache content (no re-merge - the inputs never changed) when it's missing, and CheckMergedPakStaleness reports stale so update/verify surface it. --- internal/core/merged_pak.go | 30 ++++++++++- internal/core/merged_pak_staleness_test.go | 62 ++++++++++++++++++++++ 2 files changed, 90 insertions(+), 2 deletions(-) diff --git a/internal/core/merged_pak.go b/internal/core/merged_pak.go index 1c2f342..70c4a32 100644 --- a/internal/core/merged_pak.go +++ b/internal/core/merged_pak.go @@ -195,9 +195,26 @@ func (s *Service) syncMergedPak(ctx context.Context, game *domain.Game, profileN } cachePath := gameCache.ModPath(game.ID, domain.SourceMerged, mergedPakModID, mergedPakVersion) + deployedPath := filepath.Join(game.ModPath, mergedPakFileName) if stored, ok := readMergedFingerprint(cachePath); ok { if eq, eqErr := mergedFingerprintsEqual(current, stored); eqErr == nil && eq { - return nil, nil // fast path: nothing changed + // #197 I5 fix: an unchanged fingerprint alone doesn't guarantee + // the pak is actually deployed - a PRIOR call's Install could + // have failed AFTER the fingerprint was already committed + // (wedging detection forever, since nothing here would ever + // notice), or a purge could have deliberately undeployed just + // the file while keeping the cache entry (#197 I2). Confirm + // the artifact is really on disk before trusting the fast + // path; if it's missing, redeploy the EXISTING cache content + // (self-healing) rather than re-merging - the inputs haven't + // changed, so there is nothing new to compute. + if _, statErr := os.Stat(deployedPath); statErr == nil { + return nil, nil // fast path: nothing changed, and it's actually deployed + } + if err := installer.Install(ctx, game, syntheticMod, profileName); err != nil { + return nil, fmt.Errorf("redeploying merged pak: %w", err) + } + return nil, nil } } @@ -340,7 +357,16 @@ func (s *Service) CheckMergedPakStaleness(game *domain.Game, profileName string) stored, ok := readMergedFingerprint(cachePath) if ok { if eq, eqErr := mergedFingerprintsEqual(current, stored); eqErr == nil && eq { - return nil, nil + // #197 I5 fix: mirrors syncMergedPak's identical fast-path + // check - a matching fingerprint alone doesn't prove the pak + // is actually deployed (a prior failed Install, or a purge + // that intentionally kept the cache entry, #197 I2). Without + // this, `lmm update`/`lmm verify` would report "up to date" + // for a profile whose game directory doesn't actually hold + // the merged pak at all - the exact wedge this fix closes. + if _, statErr := os.Stat(filepath.Join(game.ModPath, mergedPakFileName)); statErr == nil { + return nil, nil + } } } diff --git a/internal/core/merged_pak_staleness_test.go b/internal/core/merged_pak_staleness_test.go index 58d1c30..6e6bd82 100644 --- a/internal/core/merged_pak_staleness_test.go +++ b/internal/core/merged_pak_staleness_test.go @@ -93,3 +93,65 @@ func TestApplyMergedPakRegen_LockedModDiffStillParticipates(t *testing.T) { require.Contains(t, string(data), "locked-bear-bytes", "the locked mod's diff must still be included in the merge") require.Contains(t, string(data), "wolf-bytes") } + +// TestSyncMergedPak_FailedDeployLeg_SelfHeals is the #197 I5 regression +// test: the fingerprint is committed to the cache entry BEFORE +// installer.Install runs - if Install fails (or, equivalently here, the +// deployed file is removed by some external actor after a successful +// sync, e.g. a purge that intentionally keeps the cache entry, #197 I2), +// a matching fingerprint alone used to make every LATER syncMergedPak +// call fast-path "nothing changed" forever, even though the game +// directory doesn't actually hold the merged pak. This proves a +// subsequent sync notices the missing artifact and redeploys it WITHOUT +// re-merging (the inputs never changed - src.compileCalls must stay at 1). +func TestSyncMergedPak_FailedDeployLeg_SelfHeals(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + + _, err := svc.SyncMergedPak(context.Background(), game, "default") + require.NoError(t, err) + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + _, err = os.Stat(deployedPath) + require.NoError(t, err, "precondition: the merged pak must be deployed") + + // Simulate the deployed artifact vanishing without the cache + // entry/fingerprint changing (a failed Install that left a partial + // deploy is the same observable state - see #197 I2's purge test for + // the other real-world path into this state). + require.NoError(t, os.Remove(deployedPath)) + + srcRaw, err := svc.GetSource("fake-compiler") + require.NoError(t, err) + src, ok := srcRaw.(*fakeCompilerSource) + require.True(t, ok) + require.Equal(t, 1, src.compileCalls, "precondition: exactly one merge so far") + + _, err = svc.SyncMergedPak(context.Background(), game, "default") + require.NoError(t, err) + + _, err = os.Stat(deployedPath) + require.NoError(t, err, "a later sync must notice the missing artifact and redeploy it") + require.Equal(t, 1, src.compileCalls, "redeploying an unchanged fingerprint must NOT trigger another merge") +} + +// TestCheckMergedPakStaleness_MissingArtifact_ReportsStale is +// TestSyncMergedPak_FailedDeployLeg_SelfHeals's CHECK-side twin: `lmm +// update`/`lmm verify` call CheckMergedPakStaleness, not syncMergedPak +// directly, so it needs the identical artifact-existence confirmation - +// otherwise a wedged (fingerprint matches, file missing) profile would +// report "up to date" forever, invisible to the one safety net meant to +// catch exactly this. +func TestCheckMergedPakStaleness_MissingArtifact_ReportsStale(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + _, err := svc.SyncMergedPak(context.Background(), game, "default") + require.NoError(t, err) + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + require.NoError(t, os.Remove(deployedPath)) + + upd, err := svc.CheckMergedPakStaleness(game, "default") + require.NoError(t, err) + require.NotNil(t, upd, "a missing deployed artifact must be reported stale even though the fingerprint hasn't changed") + require.True(t, upd.RecompileNeeded) +} From c20950af3c80c1460f9d517331340b3b3a7684ff Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 21:43:53 -0400 Subject: [PATCH 81/96] fix: M1/M3/M4 minors from #197 final review M1: amend the untouched #136/#175 CHANGELOG bullet, which still claimed a .exmodz compiles into a deployable _P.pak at download time - false under the merged-only model landed by this same unreleased section. M3: fix 3 stale comments referencing retired #196 surfaces (source.Compiler, compiledFileName, Service.ApplyRecompile) that no longer exist - no dead code, just misleading doc comments. M4: applyRecompile (cmd/lmm/update.go) discarded ApplyMergedPakRegen's result.Warnings and watched for UpdateWarning/UpdateNote progress phases the function never emits (only UpdateDownloadDone) - a merge's asset-collision warnings, 'a loud warning' per the CHANGELOG, silently never printed via lmm update's single-mod apply path (TUI and DeployProfile were unaffected). --- CHANGELOG.md | 2 +- cmd/lmm/install_compile_test.go | 3 ++- cmd/lmm/update.go | 21 +++++++++++---------- cmd/lmm/update_recompile_test.go | 28 ++++++++++++++++++++++++++++ internal/source/icarus/compile.go | 3 ++- internal/source/icarus/icarus.go | 12 ++++++------ internal/tui/actions_provider.go | 14 ++++++++------ 7 files changed, 58 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aacff23..6428075 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - CLI output is now colorized by default when stdout is a terminal, extending the existing `colorGreen`/`colorRed`/`colorYellow` accent mechanism (previously only used by `deploy`/`verify`) with a full 4-color palette (green/yellow/red/cyan, plus bold/dim) across `list`, `status`, `search`, `update`, `conflicts`, and `mod show`. Table headers are bold+cyan. `lmm list` tints the whole row identically with or without `-v` (the row-tint decision is a single shared helper keyed on the mod's actual state, not the display flag): green for the common enabled+deployed case, yellow for enabled-but-undeployed, dim for disabled. `search` tints an installed mod's whole row green; `update`'s POLICY column colors per row. `status`/`mod show` color their values, not just the odd count: `lmm status -g `'s active profile and per-profile "(active)" marker are green, mod/profile counts are cyan, Link Method is cyan, Last Deploy is green (or dim when never deployed); `mod show`'s Version fields are cyan and its Update policy is colored per state (green for auto, yellow for pinned); `conflicts`' stale winner suffix is yellow; and the existing `✓`/`✗` success/failure markers extend to `update` and `mod`'s confirmation lines. Detection is TTY-aware (piped/redirected output stays plain) and layers on top of the existing `--no-color` flag and `NO_COLOR` env var (presence-only per no-color.org), which continue to work unchanged; `--json` output is never colored. Table color is applied only to already-tabwriter-padded text (accented headers, whole-row tints, or a table's genuinely last column) — never to interior cell values before they reach `text/tabwriter`, which pads columns by raw byte length and would misalign them (#112, #193) -- **Icarus built-in mod source** (`internal/source/icarus`): a public, unauthenticated Firestore-backed catalog (Project Daedalus) — `lmm search`/`install`/`update` work against it like NexusMods/CurseForge. A `.exmodz` mod file now compiles into a deployable `_P.pak` at download time via a new, game-agnostic `internal/unrealpak` PAK reader/writer and the new `deploy_mode: compile` game setting; a plain `.pak` file from the same catalog is unaffected and deploys through the existing extract/copy pipeline unchanged. Base data tables are read directly from the installed game's own `data.pak`, so a compile always matches the installed game version and works entirely offline; `internal/unrealpak` reads both the stored and the Zlib-compressed entries that pak contains, using only the standard library (#136, #175) +- **Icarus built-in mod source** (`internal/source/icarus`): a public, unauthenticated Firestore-backed catalog (Project Daedalus) — `lmm search`/`install`/`update` work against it like NexusMods/CurseForge. A `.exmodz` mod file is validated and its row-level table diffs retained via a new, game-agnostic `internal/unrealpak` PAK reader/writer and the new `deploy_mode: compile` game setting (see the merged-pak bullet below for how it actually deploys); a plain `.pak` file from the same catalog is unaffected and deploys through the existing extract/copy pipeline unchanged. Base data tables are read directly from the installed game's own `data.pak`, so a merge always matches the installed game version and works entirely offline; `internal/unrealpak` reads both the stored and the Zlib-compressed entries that pak contains, using only the standard library (#136, #175) - `lmm game detect` now recognizes Icarus (Steam App ID `1149460`) and generates a complete `games.yaml` entry for it (`deploy_mode: compile`, `sources: {icarus: icarus}`) — no more hand-editing `games.yaml` to get started. The known-games schema (`steam-games.yaml`, built-in or your own override) gained two optional fields, `deploy_mode` and `sources`, generalizing detection beyond NexusMods-only games; every existing entry is unaffected (#177) - Custom `api` sources' `search` endpoint gains `{category}`/`{tags}` path placeholders, fed from `SearchQuery.Category`/`.Tags` (URL-escaped; multiple tags comma-joined) — previously these were silently dropped with no way for a declarative source to express category/tag filtering. A definition whose `search` path omits the new placeholders is unaffected: the values are computed but never substituted in, matching today's behavior exactly (#120) - Compiled mods (`deploy_mode: compile`, e.g. Icarus) with more than one enabled `.exmodz` mod now compose correctly instead of silently shadowing each other: every enabled mod's table-row diffs are applied sequentially, in profile load order, into ONE merged `zzz_LMM_Merged_P.pak` per profile (named to mount last, so it always wins over a plain prebuilt `.pak`'s own table override) — two mods patching different fields of the same row, or entirely different rows of the same table, both survive; only a genuine same-field conflict is last-wins, and a bundled-asset path collision (which can't compose) is last-wins with a loud warning. This also fixes "the Friday problem" (a weekly base-pak refresh silently reverting a mod's patched tables, with nothing to notice): the merge regenerates whenever the enabled-mod set, load order, a mod's version, or the base pak itself changes. `lmm update` (CLI and TUI) reports a "recompile needed" row for the profile's merged pak (additive `--json` field `recompile_needed`/`reason`) alongside normal version updates; applying it regenerates and redeploys — pinned mods' diffs recompile normally, and a LOCKED mod's diff still participates in every merge (a lock pins that mod's own version, not the profile's merged pak). Installing/importing a `.exmodz` now only validates and retains it (a per-mod compiled pak is no longer generated or deployed); a plain prebuilt `.pak` mod, and every non-`deploy_mode: compile` game, is completely unaffected. `lmm verify` gains a matching "RECOMPILE NEEDED" warning row (`stale_compile`) for the profile's merged pak (#136, #175, #196, #197) diff --git a/cmd/lmm/install_compile_test.go b/cmd/lmm/install_compile_test.go index 15c976b..f63f693 100644 --- a/cmd/lmm/install_compile_test.go +++ b/cmd/lmm/install_compile_test.go @@ -33,6 +33,7 @@ type compilerInstallSource struct { *fakeInstallSource validateCalls int compileCalls int + mergeWarnings []string } // ValidateSource confirms the archive exists - this test only asserts the @@ -58,7 +59,7 @@ func (s *compilerInstallSource) MergeCompile(ctx context.Context, basePakPath st } out = append(out, data...) } - return nil, os.WriteFile(outputPath, out, 0o644) + return s.mergeWarnings, os.WriteFile(outputPath, out, 0o644) } // TestDoInstall_DeployCompile_AnnouncesRetaining guards #190 item 1: an diff --git a/cmd/lmm/update.go b/cmd/lmm/update.go index f5ee7cc..90e8faf 100644 --- a/cmd/lmm/update.go +++ b/cmd/lmm/update.go @@ -804,18 +804,19 @@ func applyUpdate(ctx context.Context, service *core.Service, game *domain.Game, // phases ApplyMergedPakRegen emits - it runs no hooks and downloads // nothing worth a progress bar). func applyRecompile(ctx context.Context, service *core.Service, game *domain.Game, profileName string) error { - progress := func(p core.DeployProgress) { - switch p.Phase { - case core.UpdateWarning: - fmt.Fprintf(os.Stderr, "Warning: %s\n", p.Detail) - case core.UpdateNote: - if verbose && !jsonOutput { - fmt.Printf(" %s\n", p.Detail) - } + result, err := service.ApplyMergedPakRegen(ctx, game, profileName, nil) + // #197 M4 fix: ApplyMergedPakRegen never emits UpdateWarning/UpdateNote + // progress events (only UpdateDownloadDone) - its merge warnings (e.g. + // an asset-path collision, "a loud warning" per the CHANGELOG) travel + // through result.Warnings instead. A progress callback watching for + // those phases would silently never fire; print result.Warnings + // directly so `lmm update`'s apply path surfaces them the same way + // DeployProfile/the TUI already do. + if result != nil { + for _, w := range result.Warnings { + fmt.Fprintf(os.Stderr, "Warning: %s\n", w) } } - - _, err := service.ApplyMergedPakRegen(ctx, game, profileName, progress) return err } diff --git a/cmd/lmm/update_recompile_test.go b/cmd/lmm/update_recompile_test.go index 4c91f19..2ccdadb 100644 --- a/cmd/lmm/update_recompile_test.go +++ b/cmd/lmm/update_recompile_test.go @@ -161,3 +161,31 @@ func TestApplySingleUpdate_Recompile_JSON(t *testing.T) { assert.Equal(t, "3.3", out.FromVersion) assert.Equal(t, "3.3", out.ToVersion) } + +// TestApplySingleUpdate_Recompile_PrintsMergeWarnings is the #197 M4 +// regression test: ApplyMergedPakRegen's merge warnings (e.g. an +// asset-path collision - "a loud warning" per the CHANGELOG) travel +// through result.Warnings, not a progress event (ApplyMergedPakRegen only +// ever emits UpdateDownloadDone) - applyRecompile used to watch for +// UpdateWarning/UpdateNote progress phases that never fire, silently +// dropping every merge warning on `lmm update`'s apply path. +func TestApplySingleUpdate_Recompile_PrintsMergeWarnings(t *testing.T) { + svc, game, compiler, _ := setupDoUpdateRecompileTest(t) + compiler.mergeWarnings = []string{"asset collision: fixture warning"} + + mod, err := svc.GetInstalledMod("fake-compiler", "bear-mount", "icarus", "default") + require.NoError(t, err) + + oldStderr := os.Stderr + r, w, pipeErr := os.Pipe() + require.NoError(t, pipeErr) + os.Stderr = w + err = applySingleUpdate(context.Background(), svc, game, mod, "default") + _ = w.Close() + os.Stderr = oldStderr + require.NoError(t, err) + + var buf bytes.Buffer + _, _ = buf.ReadFrom(r) + assert.Contains(t, buf.String(), "asset collision: fixture warning", "a merge warning must reach the CLI, not be silently dropped") +} diff --git a/internal/source/icarus/compile.go b/internal/source/icarus/compile.go index 1e71c5e..c2675c5 100644 --- a/internal/source/icarus/compile.go +++ b/internal/source/icarus/compile.go @@ -22,7 +22,8 @@ import ( // // There is no ctx parameter: every step is local file I/O over a ~2 MB pak, // with no network call and no long-running loop to cancel. The -// source.Compiler interface still takes one, for implementations that need it. +// source.MergeCompiler interface still takes one, for implementations that +// need it (MergeCompile, this package's own N-mod entry point, is one). // // The compiled pak's mount point and table-entry paths (icarusContentMountPoint, // icarusDataTablePrefix below) are Icarus-specific and deliberately live here diff --git a/internal/source/icarus/icarus.go b/internal/source/icarus/icarus.go index 76848b0..72cd876 100644 --- a/internal/source/icarus/icarus.go +++ b/internal/source/icarus/icarus.go @@ -243,12 +243,12 @@ func mapDoc(d firestoreDoc) domain.Mod { // fileNameFromURL derives a download's file name from its URL, falling back // to a synthesized "mod." name (never a bare, dot-less // fallbackExt) when the URL yields nothing usable. A dot-less fallback would -// silently defeat both isExmodzFile's case-insensitive ".exmodz" suffix -// check and compiledFileName's filepath.Ext-based rename in Service — a -// downloaded file named e.g. "exmodz" would never route through Compile. -// A parsed basename that exists but carries no extension of its own gets -// fallbackExt appended rather than being discarded outright, preserving -// whatever real name the URL offered. +// silently defeat isExmodzFile's case-insensitive ".exmodz" suffix check — +// a downloaded file named e.g. "exmodz" would never route through the +// DeployCompile ingest branch (validate+retain, #197). A parsed basename +// that exists but carries no extension of its own gets fallbackExt +// appended rather than being discarded outright, preserving whatever real +// name the URL offered. func fileNameFromURL(rawURL, fallbackExt string) string { fallback := "mod." + fallbackExt u, err := url.Parse(rawURL) diff --git a/internal/tui/actions_provider.go b/internal/tui/actions_provider.go index c49b3ce..45cf63d 100644 --- a/internal/tui/actions_provider.go +++ b/internal/tui/actions_provider.go @@ -280,17 +280,19 @@ type UpdateItem struct { // is false" contract. Locked bool LockedVersion string - // RecompileNeeded marks a #196 base-pak staleness row: a DeployCompile - // mod whose deployed compile no longer matches the game's live base - // pak. ToVersion equals FromVersion in this case - the mod itself - // hasn't changed, only the base pak has - and ApplyUpdate routes such a - // row to Service.ApplyRecompile instead of Service.ApplyUpdate. + // RecompileNeeded marks a #197 merged-pak staleness row (generalizing + // #196's per-mod version): the profile's merged pak no longer matches + // its recorded fingerprint (enabled-mod set, load order, a mod's + // version, or the base pak changed). ToVersion equals FromVersion in + // this case - u itself is the SYNTHETIC merged-pak row, not a real + // installed mod - and ApplyUpdate routes such a row to + // Service.ApplyMergedPakRegen instead of Service.ApplyUpdate. RecompileNeeded bool } // VersionLabel renders u's version change for display: the normal // " → " arrow for a real update, or "(base pak updated)" for a -// #196 RecompileNeeded row, where FromVersion == ToVersion and an arrow +// #197 RecompileNeeded row, where FromVersion == ToVersion and an arrow // would misleadingly read as a no-op. Used everywhere an UpdateItem's // version change is shown - the apply-updates modal, its result lines, and // the changelog picker/overlay - so all of them read sanely for a From 1a633c975f3ccdbd727d124cc1483706b4bd1072 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 22:32:55 -0400 Subject: [PATCH 82/96] fix: batchInstallMods (multi-select install) now syncs the merged pak (#197) Root cause of the postsmoke bug: cmd/lmm/install.go's batchInstallMods (reached from doInstall when a search returns multiple mods, installMultipleMods -> batchInstallMods) is a bespoke reimplementation of install/deploy that never went through Service.ApplyInstall - the only seam that synced the merged pak. A DeployCompile mod deploys zero files of its own (validate+retain only), so a multi-select install of 2+ exmodz mods generated the merged pak in cache but never deployed it, with nothing to warn the user - exactly the user-reported bug. Sync failures print unconditionally (not --verbose-gated) so a future failure is loud, not silent. Regression test drives the real production batchInstallMods (not a reimplementation) with two different exmodz mods and confirms the deployed merged pak contains both mods' content. --- cmd/lmm/install.go | 17 +++++++++++++ cmd/lmm/install_compile_test.go | 45 +++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/cmd/lmm/install.go b/cmd/lmm/install.go index 277f725..c94e754 100644 --- a/cmd/lmm/install.go +++ b/cmd/lmm/install.go @@ -1213,6 +1213,23 @@ func batchInstallMods(ctx context.Context, service *core.Service, game *domain.G printHookWarnings(hookErrors) + // #197 postsmoke fix (root cause): this whole function is a bespoke + // reimplementation of install/deploy that never went through + // Service.ApplyInstall, the ONLY seam that used to sync the merged pak + // - a DeployCompile game's ".exmodz" mod deploys zero files of its own + // (validate+retain only, Task 2/3), so a multi-select install left the + // merged pak generated in cache but NEVER DEPLOYED, with nothing to + // warn the user. Sync failures are printed unconditionally (not + // --verbose-gated): if this had failed loudly the first time, the user + // would have noticed immediately instead of silently missing content. + if syncWarnings, syncErr := service.SyncMergedPak(ctx, game, profileName); syncErr != nil { + fmt.Fprintf(os.Stderr, "Warning: could not sync merged pak: %v\n", syncErr) + } else { + for _, w := range syncWarnings { + fmt.Fprintf(os.Stderr, "Warning: %s\n", w) + } + } + // Summary fmt.Printf("\n--- Summary ---\n") fmt.Printf("Installed: %d\n", len(installed)) diff --git a/cmd/lmm/install_compile_test.go b/cmd/lmm/install_compile_test.go index f63f693..c7d47c9 100644 --- a/cmd/lmm/install_compile_test.go +++ b/cmd/lmm/install_compile_test.go @@ -96,3 +96,48 @@ func TestDoInstall_DeployCompile_AnnouncesRetaining(t *testing.T) { assert.Contains(t, out, "Retaining Bear_Mount.exmodz for merge...\n") assert.NotContains(t, out, "Extracting to cache...", "retaining isn't extracting - the generic message must not also print") } + +// TestBatchInstallMods_DeployCompile_DeploysMergedPak is the #197 +// postsmoke regression test: a real user's multi-select install of two +// ".exmodz" mods (the search flow's `len(selectedMods) > 1` branch, doInstall +// -> installMultipleMods -> batchInstallMods) generated the merged pak in +// CACHE but never DEPLOYED it - batchInstallMods is a bespoke +// reimplementation of install/deploy that never went through +// Service.ApplyInstall, the only seam that used to sync the merged pak. +// This drives the REAL production batchInstallMods (not a reimplementation +// or a mock) with two DIFFERENT exmodz mods and proves the merged pak is +// actually deployed on disk afterward, containing BOTH mods' content - +// exactly the class of test whose absence let this ship. +func TestBatchInstallMods_DeployCompile_DeploysMergedPak(t *testing.T) { + svc, game, src := setupDoInstallTest(t) + game.DeployMode = domain.DeployCompile + game.InstallPath = t.TempDir() + + basePak := filepath.Join(game.InstallPath, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + writeFakeBasePak(t, basePak) + + compiler := &compilerInstallSource{fakeInstallSource: src} + svc.RegisterSource(compiler) + // SyncMergedPak resolves the game's configured sources (mergeCompilerSourceForGame + // -> SourcesForGame), which requires the game to be registered - the + // production CLI always has this via withGameService's svc.GetGame, + // unlike this fixture's bare *domain.Game construction. + require.NoError(t, svc.AddGame(game)) + + bearMod := &domain.Mod{ID: "bear-mount", SourceID: "test-src", Name: "Bear Mount", Version: "1.0", GameID: "g1"} + wolfMod := &domain.Mod{ID: "wolf-mount", SourceID: "test-src", Name: "Wolf Mount", Version: "1.0", GameID: "g1"} + src.AddMod(bearMod, []domain.DownloadableFile{{ID: "bear-exmodz", Name: "Bear Mount", FileName: "Bear_Mount.exmodz", IsPrimary: true, Category: "MAIN"}}) + src.AddMod(wolfMod, []domain.DownloadableFile{{ID: "wolf-exmodz", Name: "Wolf Mount", FileName: "Wolf_Mount.exmodz", IsPrimary: true, Category: "MAIN"}}) + src.AddDownload("bear-exmodz", []byte("bear-bytes")) + src.AddDownload("wolf-exmodz", []byte("wolf-bytes")) + + err := batchInstallMods(context.Background(), svc, game, []*domain.Mod{bearMod, wolfMod}, "default") + require.NoError(t, err) + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + data, readErr := os.ReadFile(deployedPath) + require.NoError(t, readErr, "batchInstallMods must sync the merged pak - both mods deploy zero files of their own") + assert.Contains(t, string(data), "bear-bytes") + assert.Contains(t, string(data), "wolf-bytes") +} From ee028dcd591c12011a8e549e8a8fffa6340fbcd3 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 22:39:24 -0400 Subject: [PATCH 83/96] fix: sync merged pak on profile apply/sync, mod edit, and verify --fix (#197) Completes the seam audit started by the batchInstallMods fix. Each of these is a bespoke cmd-layer reimplementation (or repair path) that mutates a merge input with no seam that used to catch it: - doProfileApply: disable/enable/install-missing loops, none synced. - doProfileSync: toAdd/toRemove change profile.Mods MEMBERSHIP directly, which GetInstalledModsInProfileOrder (and so enabledExmodzSources) depends on independent of the DB Enabled flag. - doModEdit: --version is a direct regeneration trigger; a --source/--source-id relink changes the identity enabledExmodzSources keys off. - doVerify --fix: repairModVersion (moves the cache dir + recorded version) and redownloadModFile both change merge-fingerprint inputs. Each regression test drives the real production cmd-layer function (not a reimplementation) and confirms the merged pak's deployed state actually changes as a result. --- cmd/lmm/mod_edit.go | 13 ++++ cmd/lmm/mod_edit_compile_test.go | 76 +++++++++++++++++++++ cmd/lmm/profile.go | 30 ++++++++- cmd/lmm/profile_compile_test.go | 110 +++++++++++++++++++++++++++++++ cmd/lmm/verify.go | 16 +++++ cmd/lmm/verify_recompile_test.go | 36 ++++++++++ 6 files changed, 279 insertions(+), 2 deletions(-) create mode 100644 cmd/lmm/mod_edit_compile_test.go create mode 100644 cmd/lmm/profile_compile_test.go diff --git a/cmd/lmm/mod_edit.go b/cmd/lmm/mod_edit.go index fab3933..2c5cf12 100644 --- a/cmd/lmm/mod_edit.go +++ b/cmd/lmm/mod_edit.go @@ -242,6 +242,19 @@ func doModEdit(ctx context.Context, service *core.Service, game *domain.Game, cu } } + // #197 postsmoke seam-audit fix: a --version edit is a direct + // regeneration trigger; a --source/--source-id relink changes the + // identity enabledExmodzSources keys off (mod.SourceID + ":" + + // mod.ID). Sync unconditionally now that changes is non-empty - cheap + // no-op if nothing merge-relevant actually moved. + if syncWarnings, syncErr := service.SyncMergedPak(ctx, game, profileName); syncErr != nil { + fmt.Fprintf(os.Stderr, "Warning: could not sync merged pak: %v\n", syncErr) + } else { + for _, w := range syncWarnings { + fmt.Fprintf(os.Stderr, "Warning: %s\n", w) + } + } + fmt.Printf("Updated %s:\n", installedMod.Name) for _, change := range changes { fmt.Printf(" %s\n", change) diff --git a/cmd/lmm/mod_edit_compile_test.go b/cmd/lmm/mod_edit_compile_test.go new file mode 100644 index 0000000..834feb3 --- /dev/null +++ b/cmd/lmm/mod_edit_compile_test.go @@ -0,0 +1,76 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/core" + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" + "github.com/stretchr/testify/require" +) + +// TestDoModEdit_DeployCompile_VersionEditSyncsMergedPak is the #197 +// postsmoke regression test for doModEdit: a --version edit changes +// mod.Version, one of syncMergedPak's own documented regeneration +// triggers, with no seam that used to catch it. Edits to a version with +// no retained source (nothing was ever cached there) must still sync - +// proof the sync now actually runs and reacts to the mod dropping out of +// the merge (mirrors the real-world "recorded the wrong version" repair +// this flag exists for). +func TestDoModEdit_DeployCompile_VersionEditSyncsMergedPak(t *testing.T) { + configDir = t.TempDir() + dataDir = t.TempDir() + installDir := t.TempDir() + + svc, err := core.NewService(core.ServiceConfig{ConfigDir: configDir, DataDir: dataDir, CacheDir: t.TempDir()}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + writeFakeBasePak(t, basePak) + + compiler := &compilerInstallSource{fakeInstallSource: newFakeInstallSource("fake-compiler")} + svc.RegisterSource(compiler) + + game := &domain.Game{ + ID: "icarus", Name: "Icarus", InstallPath: installDir, ModPath: t.TempDir(), + DeployMode: domain.DeployCompile, LinkMethod: domain.LinkCopy, + SourceIDs: map[string]string{"fake-compiler": "external-icarus-id"}, + } + require.NoError(t, svc.AddGame(game)) + pm := getProfileManager(svc) + _, err = pm.Create(game.ID, "default") + require.NoError(t, err) + require.NoError(t, pm.SetDefault(game.ID, "default")) + + const modID, oldVersion, fileID = "bear-mount", "1.0", "exmodz-file" + gameCache := svc.GetGameCache(game) + require.NoError(t, gameCache.Store(game.ID, "fake-compiler", modID, oldVersion, cache.RetainedSourceName(fileID), []byte("bear-bytes"))) + require.NoError(t, svc.SaveInstalledMod(&domain.InstalledMod{ + Mod: domain.Mod{ID: modID, SourceID: "fake-compiler", Name: "Bear Mount", Version: oldVersion, GameID: game.ID}, + ProfileName: "default", + Enabled: true, + FileIDs: []string{fileID}, + UpdatePolicy: domain.UpdateNotify, + })) + require.NoError(t, pm.UpsertMod(game.ID, "default", domain.ModReference{SourceID: "fake-compiler", ModID: modID, Version: oldVersion, FileIDs: []string{fileID}})) + + _, err = svc.SyncMergedPak(context.Background(), game, "default") + require.NoError(t, err) + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + _, err = os.Stat(deployedPath) + require.NoError(t, err, "precondition: the merged pak must exist before the version edit") + + oldVer := editVersion + editVersion = "2.0" // no retained source exists at 2.0 + t.Cleanup(func() { editVersion = oldVer }) + + require.NoError(t, doModEdit(context.Background(), svc, game, modID)) + + _, err = os.Stat(deployedPath) + require.True(t, os.IsNotExist(err), "doModEdit --version must sync the merged pak - the mod no longer resolves under its new version and must drop out") +} diff --git a/cmd/lmm/profile.go b/cmd/lmm/profile.go index 8d139c4..276f291 100644 --- a/cmd/lmm/profile.go +++ b/cmd/lmm/profile.go @@ -561,11 +561,11 @@ func doProfileImport(ctx context.Context, service *core.Service, game *domain.Ga func runProfileSync(cmd *cobra.Command, args []string) error { return withGameService(cmd, func(ctx context.Context, service *core.Service, game *domain.Game) error { - return doProfileSync(service, game, args) + return doProfileSync(ctx, service, game, args) }) } -func doProfileSync(service *core.Service, game *domain.Game, args []string) error { +func doProfileSync(ctx context.Context, service *core.Service, game *domain.Game, args []string) error { pm := getProfileManager(service) // Determine profile name @@ -721,6 +721,19 @@ func doProfileSync(service *core.Service, game *domain.Game, args []string) erro } } + // #197 postsmoke seam-audit fix: toAdd/toRemove change profile.Mods + // MEMBERSHIP directly (AddMod/RemoveMod) - membership, not just the DB + // Enabled flag, is what GetInstalledModsInProfileOrder (and so + // enabledExmodzSources) requires, so this is a genuine merge-input + // change with no other seam to catch it. + if syncWarnings, syncErr := service.SyncMergedPak(ctx, game, profileName); syncErr != nil { + fmt.Fprintf(os.Stderr, "Warning: could not sync merged pak: %v\n", syncErr) + } else { + for _, w := range syncWarnings { + fmt.Fprintf(os.Stderr, "Warning: %s\n", w) + } + } + fmt.Printf("✓ Synced profile: %s\n", profileName) return nil } @@ -1178,6 +1191,19 @@ func doProfileApply(ctx context.Context, service *core.Service, game *domain.Gam } } + // #197 postsmoke seam-audit fix: doProfileApply is a bespoke + // disable/enable/install reimplementation - like batchInstallMods, it + // never went through a core seam that syncs the merged pak. Sync + // failures are printed unconditionally, matching batchInstallMods' + // loud-failure fix. + if syncWarnings, syncErr := service.SyncMergedPak(ctx, game, profileName); syncErr != nil { + fmt.Fprintf(os.Stderr, "Warning: could not sync merged pak: %v\n", syncErr) + } else { + for _, w := range syncWarnings { + fmt.Fprintf(os.Stderr, "Warning: %s\n", w) + } + } + fmt.Printf("\n✓ Applied profile: %s\n", profileName) return nil } diff --git a/cmd/lmm/profile_compile_test.go b/cmd/lmm/profile_compile_test.go new file mode 100644 index 0000000..0b4671b --- /dev/null +++ b/cmd/lmm/profile_compile_test.go @@ -0,0 +1,110 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" + "github.com/stretchr/testify/require" +) + +// TestDoProfileApply_DeployCompile_SyncsMergedPakOnDisable is the #197 +// postsmoke regression test for doProfileApply: it is a bespoke +// disable/enable/install reimplementation (like batchInstallMods) that +// never went through a core seam syncing the merged pak. Removing the +// LAST enabled exmodz mod from the profile (toDisable path) must undeploy +// the merged pak - proof the sync now actually runs. +func TestDoProfileApply_DeployCompile_SyncsMergedPakOnDisable(t *testing.T) { + svc, game := setupDoProfileSwitchTest(t) + game.DeployMode = domain.DeployCompile + game.InstallPath = t.TempDir() + game.SourceIDs = map[string]string{"fake-compiler": "external-icarus-id"} + require.NoError(t, svc.AddGame(game)) + + basePak := filepath.Join(game.InstallPath, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + writeFakeBasePak(t, basePak) + + compiler := &compilerInstallSource{fakeInstallSource: newFakeInstallSource("fake-compiler")} + svc.RegisterSource(compiler) + + const modID, version, fileID = "bear-mount", "1.0", "exmodz-file" + gameCache := svc.GetGameCache(game) + require.NoError(t, gameCache.Store(game.ID, "fake-compiler", modID, version, cache.RetainedSourceName(fileID), []byte("bear-bytes"))) + require.NoError(t, svc.SaveInstalledMod(&domain.InstalledMod{ + Mod: domain.Mod{ID: modID, SourceID: "fake-compiler", Name: "Bear Mount", Version: version, GameID: game.ID}, + ProfileName: "default", + Enabled: true, + FileIDs: []string{fileID}, + UpdatePolicy: domain.UpdateNotify, + })) + pm := getProfileManager(svc) + require.NoError(t, pm.UpsertMod(game.ID, "default", domain.ModReference{SourceID: "fake-compiler", ModID: modID, Version: version, FileIDs: []string{fileID}})) + + _, err := svc.SyncMergedPak(context.Background(), game, "default") + require.NoError(t, err) + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + _, err = os.Stat(deployedPath) + require.NoError(t, err, "precondition: the merged pak must exist before removing the mod from the profile") + + // Remove the mod from profile.Mods (still installed+enabled in the DB) + // - doProfileApply's toDisable path. + require.NoError(t, pm.RemoveMod(game.ID, "default", "fake-compiler", modID)) + + origYes := profileApplyYes + profileApplyYes = true + t.Cleanup(func() { profileApplyYes = origYes }) + + require.NoError(t, doProfileApply(context.Background(), svc, game, nil)) + + _, err = os.Stat(deployedPath) + require.True(t, os.IsNotExist(err), "doProfileApply must sync the merged pak when disabling the last exmodz mod") +} + +// TestDoProfileSync_DeployCompile_AddingDriftedModDeploysMergedPak is the +// #197 postsmoke regression test for doProfileSync: an installed+enabled +// exmodz mod whose profile.yaml entry drifted away (toAdd path, pm.AddMod) +// changes profile MEMBERSHIP - the input enabledExmodzSources actually +// requires - with no other seam to sync it. Proves the merged pak deploys +// once doProfileSync re-adds the mod to the profile. +func TestDoProfileSync_DeployCompile_AddingDriftedModDeploysMergedPak(t *testing.T) { + svc, game := setupDoProfileSwitchTest(t) + game.DeployMode = domain.DeployCompile + game.InstallPath = t.TempDir() + game.SourceIDs = map[string]string{"fake-compiler": "external-icarus-id"} + require.NoError(t, svc.AddGame(game)) + + basePak := filepath.Join(game.InstallPath, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + writeFakeBasePak(t, basePak) + + compiler := &compilerInstallSource{fakeInstallSource: newFakeInstallSource("fake-compiler")} + svc.RegisterSource(compiler) + + const modID, version, fileID = "bear-mount", "1.0", "exmodz-file" + gameCache := svc.GetGameCache(game) + require.NoError(t, gameCache.Store(game.ID, "fake-compiler", modID, version, cache.RetainedSourceName(fileID), []byte("bear-bytes"))) + require.NoError(t, svc.SaveInstalledMod(&domain.InstalledMod{ + Mod: domain.Mod{ID: modID, SourceID: "fake-compiler", Name: "Bear Mount", Version: version, GameID: game.ID}, + ProfileName: "default", + Enabled: true, + FileIDs: []string{fileID}, + UpdatePolicy: domain.UpdateNotify, + })) + // Deliberately NOT added to profile.Mods - simulates profile.yaml drift + // (e.g. hand-edited or lost) that doProfileSync's toAdd path exists to + // repair. + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + _, err := os.Stat(deployedPath) + require.True(t, os.IsNotExist(err), "precondition: nothing deployed yet - the mod isn't in the profile") + + require.NoError(t, doProfileSync(context.Background(), svc, game, nil)) + + data, err := os.ReadFile(deployedPath) + require.NoError(t, err, "doProfileSync must sync the merged pak after re-adding a drifted mod to the profile") + require.Equal(t, "bear-bytes", string(data)) +} diff --git a/cmd/lmm/verify.go b/cmd/lmm/verify.go index 3d47dd0..c030692 100644 --- a/cmd/lmm/verify.go +++ b/cmd/lmm/verify.go @@ -645,6 +645,22 @@ func doVerify(cmd *cobra.Command, svc *core.Service, game *domain.Game, args []s } } + // #197 postsmoke seam-audit fix: --fix can repair a VERSION MISMATCH + // (repairModVersion moves the cache dir and the recorded version) or + // redownload a file whose content has since changed upstream + // (redownloadModFile) - both are merge-fingerprint inputs with no + // other seam to catch them. No-op when --fix wasn't passed (nothing + // mutated) or the game isn't DeployCompile (SyncMergedPak's own guard). + if verifyFix { + if syncWarnings, syncErr := svc.SyncMergedPak(cmd.Context(), game, profile); syncErr != nil { + fmt.Fprintf(os.Stderr, "Warning: could not sync merged pak: %v\n", syncErr) + } else { + for _, w := range syncWarnings { + fmt.Fprintf(os.Stderr, "Warning: %s\n", w) + } + } + } + if jsonOutput { enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") diff --git a/cmd/lmm/verify_recompile_test.go b/cmd/lmm/verify_recompile_test.go index 5444a61..9a44835 100644 --- a/cmd/lmm/verify_recompile_test.go +++ b/cmd/lmm/verify_recompile_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "os" + "path/filepath" "testing" "github.com/spf13/cobra" @@ -119,3 +120,38 @@ func TestDoVerify_HealthyExmodzMod_NoFileCountMismatch(t *testing.T) { "a healthy exmodz mod (validate+retain only, zero deployment members by design) must never report file_count_mismatch") } } + +// TestDoVerify_Fix_SyncsMergedPak is the #197 postsmoke regression test +// for doVerify: --fix can repair a VERSION MISMATCH (moves the cache dir +// and the recorded version) or redownload a file whose upstream content +// changed - both are merge-fingerprint inputs with no other seam to catch +// them. This proves `lmm verify --fix` reaches the merged pak at all: a +// deployed pak that goes missing (mirrors #197 I5's self-heal scenario - +// a failed prior deploy, or a purge that intentionally kept the cache +// entry) must be redeployed by a --fix run, even though nothing else was +// broken to repair. +func TestDoVerify_Fix_SyncsMergedPak(t *testing.T) { + svc, game, _, _ := setupDoUpdateRecompileTest(t) + require.NoError(t, svc.SaveFileChecksum("fake-compiler", "bear-mount", game.ID, "default", "exmodz-file-id", "deadbeef")) + _, err := svc.SyncMergedPak(context.Background(), game, "default") + require.NoError(t, err) + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + _, err = os.Stat(deployedPath) + require.NoError(t, err, "precondition: the merged pak must be deployed") + require.NoError(t, os.Remove(deployedPath)) + + verifyProfile = "default" + verifyFix = true + t.Cleanup(func() { verifyProfile = ""; verifyFix = false }) + + cmd := &cobra.Command{} + cmd.SetContext(context.Background()) + + _ = captureStdout(t, func() error { + return doVerify(cmd, svc, game, nil) + }) + + _, err = os.Stat(deployedPath) + require.NoError(t, err, "lmm verify --fix must sync the merged pak, redeploying it when missing") +} From 6fcb4c8f11b19cc9c0a445bce49e01c16b30a648 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 22:51:18 -0400 Subject: [PATCH 84/96] fix: make merged-pak sync failures loud across every flow (#197) Task item 2 ("check the error path plumbing") surfaced a systemic issue beyond the postsmoke bug itself: syncMergedPak failures were folded into result.Notes (this codebase's --verbose-gated diagnostic channel) or appended to result.Warnings WITHOUT a corresponding live progress event, on nearly every flow that syncs - including ApplyInstall, the already-fixed single-mod install path. Several callers (doProfileSwitch, applyUpdate, doProfileImport) also simply discarded the result struct that would have carried the warning. - EnableResult/DisableResult/SwitchResult gain a Warnings field (additive) - Notes alone had no unconditional display channel. - EnableMod/DisableMod/UninstallMod/ApplyProfileSwitch/ApplyRollback/ PurgeProfile route sync failures through Warnings instead of Notes. - ApplyInstall/ApplyUpdate/ApplyRollback/PurgeProfile additionally emit a live *Warning progress event, so a caller driving purely off progress (like doInstall/applyUpdate) sees it without needing to read the result struct at all. - cmd/lmm/mod.go (enable/disable), cmd/lmm/profile.go (switch/import) now print result.Warnings unconditionally to stderr. - internal/tui/service_core.go folds the new/now-populated Warnings fields into ActionOutcome.Warnings for enable/disable/switch (the TUI's own equivalent of loud), so nothing silently regressed there. ReorderProfileMods (bare-error signature, discards sync warnings) is deliberately left as-is - its own doc comment already reasons about this tradeoff explicitly, and lmm update/verify remain the safety net. Regression test drives a real single-mod install through a forced merge failure and confirms it reaches stderr unconditionally - the exact scenario the task named ("if install's sync deploy had failed loudly the user would have seen it"). --- cmd/lmm/install_compile_test.go | 43 ++++++++++++++ cmd/lmm/mod.go | 14 +++++ cmd/lmm/profile.go | 17 +++++- internal/core/flows.go | 101 ++++++++++++++++++++++++-------- internal/tui/service_core.go | 13 +++- 5 files changed, 160 insertions(+), 28 deletions(-) diff --git a/cmd/lmm/install_compile_test.go b/cmd/lmm/install_compile_test.go index c7d47c9..f7e67d7 100644 --- a/cmd/lmm/install_compile_test.go +++ b/cmd/lmm/install_compile_test.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "context" "os" "path/filepath" @@ -34,6 +35,7 @@ type compilerInstallSource struct { validateCalls int compileCalls int mergeWarnings []string + mergeErr error } // ValidateSource confirms the archive exists - this test only asserts the @@ -51,6 +53,9 @@ func (s *compilerInstallSource) ValidateSource(sourceFilePath string) error { // internal/core/service_icarus_compile_test.go's fakeCompilerSource). func (s *compilerInstallSource) MergeCompile(ctx context.Context, basePakPath string, sources []source.MergeSource, outputPath string) ([]string, error) { s.compileCalls++ + if s.mergeErr != nil { + return nil, s.mergeErr + } var out []byte for _, src := range sources { data, err := os.ReadFile(src.ExmodzPath) @@ -141,3 +146,41 @@ func TestBatchInstallMods_DeployCompile_DeploysMergedPak(t *testing.T) { assert.Contains(t, string(data), "bear-bytes") assert.Contains(t, string(data), "wolf-bytes") } + +// TestDoInstall_DeployCompile_SyncFailure_PrintsLoudly is the #197 +// postsmoke "must be LOUD" regression test: ApplyInstall's own sync call +// used to only append to result.Warnings, which doInstall (the single-mod +// install path) never reads back - a sync failure here was completely +// silent, not even --verbose-gated, the exact plumbing gap the task asked +// to check. Proves a merge failure during a real single-mod install +// reaches stderr unconditionally. +func TestDoInstall_DeployCompile_SyncFailure_PrintsLoudly(t *testing.T) { + svc, game, src := setupDoInstallTest(t) + game.DeployMode = domain.DeployCompile + game.InstallPath = t.TempDir() + require.NoError(t, svc.AddGame(game)) + + basePak := filepath.Join(game.InstallPath, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + writeFakeBasePak(t, basePak) + + compiler := &compilerInstallSource{fakeInstallSource: src, mergeErr: assert.AnError} + svc.RegisterSource(compiler) + + src.AddMod(&domain.Mod{ID: "mod1", SourceID: "test-src", Name: "Bear Mount", Version: "1.0", GameID: "g1"}, + []domain.DownloadableFile{{ID: "main", Name: "Bear Mount", FileName: "Bear_Mount.exmodz", IsPrimary: true, Category: "MAIN"}}) + src.AddDownload("main", []byte("fake-exmodz-bytes")) + + oldStderr := os.Stderr + r, w, pipeErr := os.Pipe() + require.NoError(t, pipeErr) + os.Stderr = w + err := doInstall(context.Background(), svc, game, nil) + _ = w.Close() + os.Stderr = oldStderr + require.NoError(t, err, "a merge failure is non-fatal to the install itself - the mod is still validated+retained+recorded") + + var buf bytes.Buffer + _, _ = buf.ReadFrom(r) + assert.Contains(t, buf.String(), "Warning:", "a merge failure during install must print a Warning unconditionally, not silently vanish into a discarded result") +} diff --git a/cmd/lmm/mod.go b/cmd/lmm/mod.go index 5ece4c6..80ef4ff 100644 --- a/cmd/lmm/mod.go +++ b/cmd/lmm/mod.go @@ -411,10 +411,12 @@ func doModEnable(ctx context.Context, service *core.Service, game *domain.Game, // the result struct, exactly like doModDisable below. if result != nil { printModNotes(result.Notes) + printModWarnings(result.Warnings) } return err } printModNotes(result.Notes) + printModWarnings(result.Warnings) if !result.Changed { fmt.Printf("%s is already enabled.\n", mod.Name) @@ -462,10 +464,12 @@ func doModDisable(ctx context.Context, service *core.Service, game *domain.Game, // before it could allocate the result struct. if result != nil { printModNotes(result.Notes) + printModWarnings(result.Warnings) } return err } printModNotes(result.Notes) + printModWarnings(result.Warnings) if !result.Changed { fmt.Printf("%s is already disabled.\n", mod.Name) @@ -495,6 +499,16 @@ func printModNotes(notes []string) { } } +// printModWarnings prints EnableResult.Warnings/DisableResult.Warnings +// unconditionally to stderr (#197 postsmoke fix) - unlike printModNotes, +// these must reach the user regardless of --verbose. Today the only +// producer is a merged-pak sync failure; nil-safe like printModNotes. +func printModWarnings(warnings []string) { + for _, w := range warnings { + fmt.Fprintf(os.Stderr, "Warning: %s\n", w) + } +} + func runModFiles(cmd *cobra.Command, args []string) error { return withGameService(cmd, func(ctx context.Context, svc *core.Service, game *domain.Game) error { return doModFiles(svc, game, args[0]) diff --git a/cmd/lmm/profile.go b/cmd/lmm/profile.go index 276f291..3bfcae9 100644 --- a/cmd/lmm/profile.go +++ b/cmd/lmm/profile.go @@ -378,12 +378,19 @@ func doProfileSwitch(ctx context.Context, service *core.Service, game *domain.Ga } } - if _, err := service.ApplyProfileSwitch(ctx, game, plan, progress); err != nil { + result, err := service.ApplyProfileSwitch(ctx, game, plan, progress) + if err != nil { // Diagnostics accumulated before a fatal error (ApplyProfileSwitch's // error-path convention returns them alongside it) were already // printed above, live, via progress - nothing left to print here. return err } + // #197 postsmoke fix: SwitchResult.Warnings (unconditional stderr, + // unlike .Notes above) - today, only a merged-pak sync failure for the + // target profile. Previously this whole result was discarded. + for _, w := range result.Warnings { + fmt.Fprintf(os.Stderr, "Warning: %s\n", w) + } fmt.Printf("\n✓ Switched to profile: %s\n", targetName) return nil @@ -537,6 +544,14 @@ func doProfileImport(ctx context.Context, service *core.Service, game *domain.Ga return err } + // #197 postsmoke fix: result.Warnings was never read - a merged-pak + // sync failure only ever reached the ImportNote progress event above + // (--verbose-gated), so it was silent by default. Print unconditionally + // as the loud backstop, matching applyRecompile's identical fix (M4). + for _, w := range result.Warnings { + fmt.Fprintf(os.Stderr, "Warning: %s\n", w) + } + switch { case profileImportNoInstall: if result.Skipped > 0 { diff --git a/internal/core/flows.go b/internal/core/flows.go index 4e6d45a..7138453 100644 --- a/internal/core/flows.go +++ b/internal/core/flows.go @@ -57,6 +57,13 @@ func (s *Service) ReorderProfileMods(gameID, profileName string, mods []domain.M type EnableResult struct { Changed bool Notes []string + // Warnings holds diagnostics that must reach the user unconditionally + // (#197 postsmoke fix), unlike Notes' --verbose-only display contract - + // today, only a merged-pak sync failure. A silent sync failure here is + // exactly the class of bug the postsmoke fix-wave exists to close: the + // mod's Enabled bit flips, but the game directory may not actually + // reflect it. + Warnings []string } // DisableResult reports the outcome of DisableMod. Changed mirrors @@ -70,6 +77,9 @@ type EnableResult struct { type DisableResult struct { Changed bool Notes []string + // Warnings mirrors EnableResult.Warnings' identical rationale + // (#197 postsmoke fix): unconditional display, unlike Notes. + Warnings []string } // EnableMod deploys an installed-but-disabled mod's files from the cache to @@ -118,12 +128,13 @@ func (s *Service) EnableMod(ctx context.Context, game *domain.Game, profileName, return result, fmt.Errorf("failed to update mod status: %w", err) } + // #197 postsmoke fix: Warnings, not Notes - Notes is --verbose-gated in + // the CLI (printModNotes), so a sync failure here used to be silent by + // default. if syncWarnings, syncErr := s.syncMergedPak(ctx, game, profileName); syncErr != nil { - result.Notes = append(result.Notes, fmt.Sprintf("Warning: could not sync merged pak: %v", syncErr)) + result.Warnings = append(result.Warnings, fmt.Sprintf("could not sync merged pak: %v", syncErr)) } else { - for _, w := range syncWarnings { - result.Notes = append(result.Notes, "Warning: "+w) - } + result.Warnings = append(result.Warnings, syncWarnings...) } result.Changed = true @@ -199,12 +210,11 @@ func (s *Service) DisableMod(ctx context.Context, game *domain.Game, profileName return result, fmt.Errorf("failed to update mod status: %w", err) } + // #197 postsmoke fix: Warnings, not Notes (see EnableMod's identical fix). if syncWarnings, syncErr := s.syncMergedPak(ctx, game, profileName); syncErr != nil { - result.Notes = append(result.Notes, fmt.Sprintf("Warning: could not sync merged pak: %v", syncErr)) + result.Warnings = append(result.Warnings, fmt.Sprintf("could not sync merged pak: %v", syncErr)) } else { - for _, w := range syncWarnings { - result.Notes = append(result.Notes, "Warning: "+w) - } + result.Warnings = append(result.Warnings, syncWarnings...) } result.Changed = true @@ -336,12 +346,14 @@ func (s *Service) UninstallMod(ctx context.Context, game *domain.Game, profileNa result.Warnings = append(result.Warnings, fmt.Sprintf("uninstall.after_all hook failed: %v", err)) } + // #197 postsmoke fix: UninstallResult.Warnings (unconditional stderr) + // already exists for exactly this - Notes is --verbose-gated + // (printUninstallDiagnostics), so a sync failure here used to be + // silent by default. if syncWarnings, syncErr := s.syncMergedPak(ctx, game, profileName); syncErr != nil { - result.Notes = append(result.Notes, fmt.Sprintf("Warning: could not sync merged pak: %v", syncErr)) + result.Warnings = append(result.Warnings, fmt.Sprintf("could not sync merged pak: %v", syncErr)) } else { - for _, w := range syncWarnings { - result.Notes = append(result.Notes, "Warning: "+w) - } + result.Warnings = append(result.Warnings, syncWarnings...) } return result, nil @@ -2245,8 +2257,16 @@ func (s *Service) PurgeProfile(ctx context.Context, game *domain.Game, profileNa // #197 I2 fix: see PurgeMergedPak's own doc comment - exmodz mods have // no per-mod deployment for the loop above to have already undeployed. + // #197 postsmoke fix: Warnings, not Notes, AND emit PurgeWarning - + // cmd/lmm/purge.go's own doc comment claims every Notes/Warnings entry + // has a corresponding live event; this one didn't, so it was + // completely invisible (not even --verbose-gated). if perr := s.PurgeMergedPak(ctx, game, profileName, opts.Uninstall); perr != nil { - result.Notes = append(result.Notes, fmt.Sprintf("Warning: could not remove merged pak: %v", perr)) + msg := fmt.Sprintf("could not remove merged pak: %v", perr) + result.Warnings = append(result.Warnings, msg) + if progress != nil { + progress(DeployProgress{Phase: PurgeWarning, Detail: msg}) + } } return result, nil @@ -2445,6 +2465,10 @@ func (s *Service) PlanProfileSwitch(ctx context.Context, game *domain.Game, targ type SwitchResult struct { Disabled, Enabled, Installed int Notes []string + // Warnings holds diagnostics that must reach the user unconditionally + // (#197 postsmoke fix), unlike Notes' --verbose-only display contract - + // today, only a merged-pak sync failure for plan.To. + Warnings []string } // ApplyProfileSwitch executes a plan produced by PlanProfileSwitch: disables @@ -2717,12 +2741,13 @@ func (s *Service) ApplyProfileSwitch(ctx context.Context, game *domain.Game, pla return result, fmt.Errorf("setting default profile: %w", err) } + // #197 postsmoke fix: Warnings, not Notes - SwitchResult.Notes is + // --verbose-gated in the CLI, so a sync failure here used to be + // silent by default. if syncWarnings, syncErr := s.syncMergedPak(ctx, game, plan.To); syncErr != nil { - result.Notes = append(result.Notes, fmt.Sprintf("Warning: could not sync merged pak: %v", syncErr)) + result.Warnings = append(result.Warnings, fmt.Sprintf("could not sync merged pak: %v", syncErr)) } else { - for _, w := range syncWarnings { - result.Notes = append(result.Notes, "Warning: "+w) - } + result.Warnings = append(result.Warnings, syncWarnings...) } return result, nil @@ -3731,10 +3756,21 @@ func (s *Service) ApplyInstall(ctx context.Context, game *domain.Game, plan *Ins emit(w) } + // #197 postsmoke fix: appending to result.Warnings alone is not loud - + // doInstall (cmd/lmm) never reads result.Warnings back, only the + // progress events emitted live above (InstallWarning is what actually + // reaches stderr). A sync failure here used to be completely silent, + // the exact plumbing gap that let the postsmoke bug through even on + // the already-fixed single-mod install path. if syncWarnings, syncErr := s.syncMergedPak(ctx, game, plan.Profile); syncErr != nil { - result.Warnings = append(result.Warnings, fmt.Sprintf("syncing merged pak: %v", syncErr)) + msg := fmt.Sprintf("syncing merged pak: %v", syncErr) + result.Warnings = append(result.Warnings, msg) + emit(DeployProgress{Phase: InstallWarning, Detail: msg}) } else { - result.Warnings = append(result.Warnings, syncWarnings...) + for _, w := range syncWarnings { + result.Warnings = append(result.Warnings, w) + emit(DeployProgress{Phase: InstallWarning, Detail: w}) + } } return result, nil @@ -4560,10 +4596,21 @@ func (s *Service) ApplyUpdate(ctx context.Context, game *domain.Game, profileNam result.Applied = append(result.Applied, fmt.Sprintf("%s %s → %s", mod.Name, mod.Version, effectiveVersion)) + // #197 postsmoke fix: also emit UpdateWarning - appending to + // result.Warnings alone is not loud enough, since applyUpdate + // (cmd/lmm/update.go) discards ApplyUpdate's result entirely + // (`_, err := ...`) and drives its console output purely from live + // progress events, exactly the plumbing gap the ApplyInstall fix + // closed for install. if syncWarnings, syncErr := s.syncMergedPak(ctx, game, profileName); syncErr != nil { - result.Warnings = append(result.Warnings, fmt.Sprintf("syncing merged pak: %v", syncErr)) + msg := fmt.Sprintf("syncing merged pak: %v", syncErr) + result.Warnings = append(result.Warnings, msg) + emit(DeployProgress{Phase: UpdateWarning, Detail: msg}) } else { - result.Warnings = append(result.Warnings, syncWarnings...) + for _, w := range syncWarnings { + result.Warnings = append(result.Warnings, w) + emit(DeployProgress{Phase: UpdateWarning, Detail: w}) + } } return result, nil @@ -4820,12 +4867,18 @@ func (s *Service) ApplyRollback(ctx context.Context, game *domain.Game, profileN // #197 I1 fix: a rollback changes the mod's Version (and possibly its // FileIDs), both regeneration triggers - without this, the merged pak // keeps the rolled-away-from version's diff until some OTHER flow - // happens to sync it. + // happens to sync it. #197 postsmoke fix: Warnings, not Notes (Notes is + // --verbose-gated in the CLI) - AND emit UpdateWarning: doUpdateRollback + // (cmd/lmm/update.go) drives its console output from live progress + // events, never reads RollbackResult.Warnings back directly. if syncWarnings, syncErr := s.syncMergedPak(ctx, game, profileName); syncErr != nil { - result.Notes = append(result.Notes, fmt.Sprintf("Warning: could not sync merged pak: %v", syncErr)) + msg := fmt.Sprintf("could not sync merged pak: %v", syncErr) + result.Warnings = append(result.Warnings, msg) + emit(DeployProgress{Phase: UpdateWarning, Detail: msg}) } else { for _, w := range syncWarnings { - result.Notes = append(result.Notes, "Warning: "+w) + result.Warnings = append(result.Warnings, w) + emit(DeployProgress{Phase: UpdateWarning, Detail: w}) } } diff --git a/internal/tui/service_core.go b/internal/tui/service_core.go index 0550b50..6a37a38 100644 --- a/internal/tui/service_core.go +++ b/internal/tui/service_core.go @@ -735,7 +735,9 @@ func (p *coreProvider) EnableMod(ctx context.Context, item ModItem) (ActionOutco if !result.Changed { return ActionOutcome{Message: fmt.Sprintf("%q is already enabled", item.Name)}, nil } - return ActionOutcome{Message: fmt.Sprintf("Enabled %q", item.Name), Warnings: mergeDiagnostics(nil, result.Notes)}, nil + // #197 postsmoke fix: fold in result.Warnings (a merged-pak sync + // failure lands there now, not result.Notes). + return ActionOutcome{Message: fmt.Sprintf("Enabled %q", item.Name), Warnings: mergeDiagnostics(result.Warnings, result.Notes)}, nil } func (p *coreProvider) DisableMod(ctx context.Context, item ModItem) (ActionOutcome, error) { @@ -746,7 +748,9 @@ func (p *coreProvider) DisableMod(ctx context.Context, item ModItem) (ActionOutc if !result.Changed { return ActionOutcome{Message: fmt.Sprintf("%q is already disabled", item.Name)}, nil } - return ActionOutcome{Message: fmt.Sprintf("Disabled %q", item.Name), Warnings: mergeDiagnostics(nil, result.Notes)}, nil + // #197 postsmoke fix: fold in result.Warnings (a merged-pak sync + // failure lands there now, not result.Notes). + return ActionOutcome{Message: fmt.Sprintf("Disabled %q", item.Name), Warnings: mergeDiagnostics(result.Warnings, result.Notes)}, nil } // UninstallMod runs the same hook configuration cmd/lmm/uninstall.go's @@ -1075,9 +1079,12 @@ func (p *coreProvider) ApplyProfileSwitch(ctx context.Context, profileName strin if err != nil { return ActionOutcome{}, fmt.Errorf("switching to %s: %w", profileName, err) } + // #197 postsmoke fix: fold in result.Warnings (a merged-pak sync + // failure now lands there, not result.Notes - SwitchResult gained a + // Warnings field for exactly this). return ActionOutcome{ Message: fmt.Sprintf("Switched to %q", profileName), - Warnings: mergeDiagnostics(installFailures, result.Notes), + Warnings: mergeDiagnostics(append(installFailures, result.Warnings...), result.Notes), }, nil } From 94a34a7b2e0ed8aceb93fcdaf3d52b7ce162b088 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 23:05:55 -0400 Subject: [PATCH 85/96] fix: install/import UX wording and stale-reason accuracy for merged pak (#197) Closes out the remaining postsmoke UX corrections beyond the sync-plumbing fixes: "0 files" read as a failure for a validate+retain-only exmodz mod, `mod files` gave false "may need to be redeployed" guidance for the same mods, and `verify`'s RECOMPILE NEEDED row always blamed "base pak updated" even when the real cause was a missing artifact, then pointed at a bare `lmm update` that (being notify-policy by construction) only reports the row again instead of fixing it. - doInstall/doInstallBatch/batchInstallMods/doImport now print "Installed (merged pak updated)" instead of "(0 files)"/"Files deployed: 0" when a DeployCompile mod deploys zero files by design. - `lmm mod files` explains that a zero-file DeployCompile mod participates in the profile's merged pak, reusing verify.go's hasRetainedSource to distinguish that case from a genuinely broken record. - domain.Update gains RecompileReason ("base pak updated" | "not deployed"), set by CheckMergedPakStaleness from the same fingerprint/ artifact-existence check that already distinguished the two internally (#197 I5). `lmm verify` surfaces the real reason in both text and --json (the `note` field), and the fix hint now says `lmm update --all`. Regression tests added/extended for every site: install/import output text, mod files' two branches (exmodz vs a genuinely broken record), and both CheckMergedPakStaleness reason values at the core and CLI layers. docs/man/man1/lmm-verify.1 regenerated (`make man`) for the corrected RECOMPILE NEEDED help text. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 1 + cmd/lmm/import.go | 9 +- cmd/lmm/import_compile_test.go | 4 +- cmd/lmm/install.go | 30 +++++- cmd/lmm/install_compile_test.go | 10 +- cmd/lmm/mod.go | 6 ++ cmd/lmm/mod_files_compile_test.go | 108 +++++++++++++++++++++ cmd/lmm/verify.go | 17 +++- cmd/lmm/verify_recompile_test.go | 33 +++++++ docs/man/man1/lmm-verify.1 | 7 +- internal/core/merged_pak.go | 8 ++ internal/core/merged_pak_staleness_test.go | 2 + internal/domain/mod.go | 7 ++ 13 files changed, 228 insertions(+), 14 deletions(-) create mode 100644 cmd/lmm/mod_files_compile_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 6428075..f95c851 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `Service.GetEffectiveLinkMethod` — the profile > game > global resolution behind every deploy/install/import/status/verify operation — no longer silently swallows an invalid profile `link_method`: since #172, `config.LoadProfile` fails loud on an unrecognized value, but `GetEffectiveLinkMethod` treated ANY profile-load error, including that one, as "no explicit override" and fell back to the game/global default with nothing surfaced — meaning a hand-edited profile with a typo'd `link_method` could deploy with the wrong method, no error, no warning. It now distinguishes that case (`errors.Is(err, domain.ErrInvalidLinkMethod)`) from a missing/unreadable profile file — which still degrades silently by design, since profiles are optional — and returns the validation error instead, propagated through every call site (`DeployProfile`, `EnableMod`/`DisableMod`, `ApplyInstall`/`ApplyUpdate`/`ApplyRollback`/`ApplyImport`/`ApplyProfileSwitch`, `lmm deploy`/`import`/`install`/`profile apply`/`verify --fix`). Narrow in practice — both save and load paths validate now, so only a profile hand-edited after the fact can trigger it — but it closes the one place #172's fail-loud contract didn't reach (#189) - `lmm import` of a local `.exmodz` file for a `deploy_mode: compile` game (Icarus) now routes through the same compile step as a download: it resolves the game's mapped `source.Compiler`-capable source from the registry and compiles the archive against the installed base pak, caching the resulting `_P.pak` the same way `DownloadModToCache` does. Previously the import path extracted/copied `.exmodz` files as-is, landing an uncompiled archive in the cache instead of a deployable pak. A missing compiler-capable source or missing base pak now fails loud with an actionable error rather than silently caching the uncompiled file; non-`.exmodz` imports are unaffected (#173) - `lmm mod disable` undeployed a mod's files and cleared `enabled`, but never cleared `deployed` — `lmm list -v` kept showing DEPLOYED yes after disable. The disable flow now clears `deployed` unconditionally after the undeploy attempt, even when the undeploy itself only partially succeeds (already a non-fatal, Note-reported condition), so the flag always reflects disable-intent rather than lagging behind a best-effort file cleanup. The symmetric enable path had the same gap — enabling a disabled mod re-deployed its files without ever setting `deployed` back to true — and is fixed the same way. Both `SetModDeployed` calls follow the same non-fatal Note convention already used by `DeployProfile`/`PurgeProfile` for this same setter: a failure to record the flag doesn't block the primary enable/disable outcome (#183) +- `lmm install`'s multi-select path (`batchInstallMods`, used whenever more than one search result is selected) never synced the profile's merged pak for a `deploy_mode: compile` game (Icarus): installing `.exmodz` mods this way validated and cached them but left the merged pak undeployed, silently, until a later `lmm update` self-healed it. A seam audit of every mutation entry point that can change a profile's enabled-exmodz set/order/versions found and closed 4 more gaps with the same shape — `lmm profile apply`/`profile sync`, `lmm mod edit --version`, and `lmm verify --fix` also never reached the merged-pak sync. All five now call the same shared `Service.SyncMergedPak` entry point the rest of the flows already used, instead of bypassing it. Separately, several flows that _did_ call the sync already (including single-mod `lmm install`) only recorded a failure in a diagnostic field their own CLI caller never read back — a sync failure could be completely silent, not even `--verbose`-gated. Sync failures now surface unconditionally on stderr everywhere the merged pak is synced (CLI and TUI). Also corrected three misleading messages this bug produced along the way: `lmm install`/`import` say "Installed (merged pak updated)" instead of a false "(0 files)" for a compile-mode mod (deploying zero files of its own is correct, not a failure); `lmm mod files` explains that such a mod participates in the profile's merged pak instead of suggesting it "may need to be redeployed"; and `lmm verify`'s "RECOMPILE NEEDED" row now names the real cause (a changed base pak vs. an artifact simply missing from disk) and points at `lmm update --all`, since this row is never auto-applied by a bare `lmm update` (#197) ## [1.27.1] - 2026-07-30 diff --git a/cmd/lmm/import.go b/cmd/lmm/import.go index b537b6b..96e37e0 100644 --- a/cmd/lmm/import.go +++ b/cmd/lmm/import.go @@ -416,7 +416,14 @@ func doImport(ctx context.Context, cmd *cobra.Command, service *core.Service, ga printHookWarnings(hookErrors) fmt.Printf("\n✓ Imported: %s\n", result.Mod.Name) - fmt.Printf(" Files deployed: %d\n", result.FilesExtracted) + // #197 postsmoke UX fix: see doInstall's identical fix (cmd/lmm/install.go) + // - a DeployCompile ".exmodz" mod deploys zero files of its own by + // design (validate+retain only). + if game.DeployMode == domain.DeployCompile && result.FilesExtracted == 0 { + fmt.Println(" Installed (merged pak updated)") + } else { + fmt.Printf(" Files deployed: %d\n", result.FilesExtracted) + } fmt.Printf(" Added to profile: %s\n", profileName) if result.LinkedSource == domain.SourceLocal { diff --git a/cmd/lmm/import_compile_test.go b/cmd/lmm/import_compile_test.go index d3bc80d..4c2e7fe 100644 --- a/cmd/lmm/import_compile_test.go +++ b/cmd/lmm/import_compile_test.go @@ -79,10 +79,12 @@ func TestDoImport_DeployCompile_ImportedModParticipatesInMerge(t *testing.T) { archivePath := filepath.Join(t.TempDir(), "Bear_Mount.exmodz") require.NoError(t, os.WriteFile(archivePath, []byte("bear-exmodz-bytes"), 0o644)) - _, err := captureStdoutErr(t, func() error { + out, err := captureStdoutErr(t, func() error { return doImport(context.Background(), &cobra.Command{}, svc, game, []string{archivePath}) }) require.NoError(t, err) + require.Contains(t, out, "Installed (merged pak updated)", + "#197 postsmoke UX fix: a zero-file exmodz import must say what happened, not print the misleading 'Files deployed: 0'") prof, err := svc.NewProfileManager().Get(game.ID, "default") require.NoError(t, err) diff --git a/cmd/lmm/install.go b/cmd/lmm/install.go index c94e754..556da74 100644 --- a/cmd/lmm/install.go +++ b/cmd/lmm/install.go @@ -645,7 +645,16 @@ func doInstall(ctx context.Context, service *core.Service, game *domain.Game, ar } fmt.Printf("\n✓ Installed: %s v%s\n", mod.Name, mod.Version) - fmt.Printf(" Files deployed: %d\n", result.FilesDeployed) + // #197 postsmoke UX fix: a DeployCompile ".exmodz" mod deploys zero + // files of its own by design (validate+retain only - it participates + // in the profile's shared merged pak instead, synced separately + // above) - "Files deployed: 0" read as a failure, not the correct, + // expected outcome it actually is. + if game.DeployMode == domain.DeployCompile && result.FilesDeployed == 0 { + fmt.Println(" Installed (merged pak updated)") + } else { + fmt.Printf(" Files deployed: %d\n", result.FilesDeployed) + } fmt.Printf(" Added to profile: %s\n", profileName) return nil @@ -720,7 +729,14 @@ func doInstallBatch(ctx context.Context, service *core.Service, game *domain.Gam case core.InstallDepConflictWarning: fmt.Printf(" ⚠ %s\n", p.Detail) case core.InstallDepInstalled: - fmt.Printf(" ✓ Installed (%d files)\n", p.FilesExtracted) + // #197 postsmoke UX fix: see the single-mod path's identical + // fix above - a DeployCompile ".exmodz" dependency deploys + // zero files of its own by design. + if game.DeployMode == domain.DeployCompile && p.FilesExtracted == 0 { + fmt.Println(" ✓ Installed (merged pak updated)") + } else { + fmt.Printf(" ✓ Installed (%d files)\n", p.FilesExtracted) + } case core.InstallNote: if verbose { fmt.Printf(" %s\n", p.Detail) @@ -1194,7 +1210,15 @@ func batchInstallMods(ctx context.Context, service *core.Service, game *domain.G fmt.Printf(" Warning: could not update profile: %v\n", err) } - fmt.Printf(" ✓ Installed (%d files)\n", downloadResult.FilesExtracted) + // #197 postsmoke UX fix: see doInstall's identical fix - a + // DeployCompile ".exmodz" mod deploys zero files of its own by + // design (validate+retain only; the merged pak sync below is what + // actually deploys it). + if game.DeployMode == domain.DeployCompile && downloadResult.FilesExtracted == 0 { + fmt.Println(" ✓ Installed (merged pak updated)") + } else { + fmt.Printf(" ✓ Installed (%d files)\n", downloadResult.FilesExtracted) + } installed = append(installed, mod.Name) // Run install.after_each hook diff --git a/cmd/lmm/install_compile_test.go b/cmd/lmm/install_compile_test.go index f7e67d7..e7e5200 100644 --- a/cmd/lmm/install_compile_test.go +++ b/cmd/lmm/install_compile_test.go @@ -5,6 +5,7 @@ import ( "context" "os" "path/filepath" + "strings" "testing" "github.com/DonovanMods/linux-mod-manager/internal/domain" @@ -100,6 +101,8 @@ func TestDoInstall_DeployCompile_AnnouncesRetaining(t *testing.T) { assert.Equal(t, 1, compiler.validateCalls) assert.Contains(t, out, "Retaining Bear_Mount.exmodz for merge...\n") assert.NotContains(t, out, "Extracting to cache...", "retaining isn't extracting - the generic message must not also print") + assert.Contains(t, out, "Installed (merged pak updated)", + "#197 postsmoke UX fix: 'Files deployed: 0' read as a failure, not the correct expected outcome for a validate+retain-only exmodz mod") } // TestBatchInstallMods_DeployCompile_DeploysMergedPak is the #197 @@ -137,7 +140,9 @@ func TestBatchInstallMods_DeployCompile_DeploysMergedPak(t *testing.T) { src.AddDownload("bear-exmodz", []byte("bear-bytes")) src.AddDownload("wolf-exmodz", []byte("wolf-bytes")) - err := batchInstallMods(context.Background(), svc, game, []*domain.Mod{bearMod, wolfMod}, "default") + out, err := captureStdoutErr(t, func() error { + return batchInstallMods(context.Background(), svc, game, []*domain.Mod{bearMod, wolfMod}, "default") + }) require.NoError(t, err) deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") @@ -145,6 +150,9 @@ func TestBatchInstallMods_DeployCompile_DeploysMergedPak(t *testing.T) { require.NoError(t, readErr, "batchInstallMods must sync the merged pak - both mods deploy zero files of their own") assert.Contains(t, string(data), "bear-bytes") assert.Contains(t, string(data), "wolf-bytes") + + assert.Equal(t, 2, strings.Count(out, "Installed (merged pak updated)"), + "#197 postsmoke UX fix: each zero-file exmodz mod must say what happened, not print the misleading '(0 files)'") } // TestDoInstall_DeployCompile_SyncFailure_PrintsLoudly is the #197 diff --git a/cmd/lmm/mod.go b/cmd/lmm/mod.go index 80ef4ff..34b1098 100644 --- a/cmd/lmm/mod.go +++ b/cmd/lmm/mod.go @@ -541,6 +541,12 @@ func doModFiles(svc *core.Service, game *domain.Game, modID string) error { fmt.Printf("Files deployed by %s (%s):\n\n", mod.Name, modID) if len(files) == 0 { + gameCache := svc.GetGameCache(game) + if game.DeployMode == domain.DeployCompile && hasRetainedSource(gameCache, game.ID, mod.SourceID, modID, mod.Version, mod.FileIDs) { + fmt.Println(" No files of its own - this mod participates in the profile's merged pak.") + fmt.Printf(" (See zzz_LMM_Merged_P.pak; run `lmm verify` to check the merged pak is up to date)\n") + return nil + } fmt.Println(" No deployed files tracked.") fmt.Println(" (Files are tracked on install; existing mods may need to be redeployed)") return nil diff --git a/cmd/lmm/mod_files_compile_test.go b/cmd/lmm/mod_files_compile_test.go new file mode 100644 index 0000000..76a079c --- /dev/null +++ b/cmd/lmm/mod_files_compile_test.go @@ -0,0 +1,108 @@ +package main + +import ( + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/core" + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestDoModFiles_DeployCompile_ExmodzModExplainsMergedPak is the #197 +// postsmoke UX fix: `lmm mod files ` for a validated+retained ".exmodz" +// mod (zero deployed files by design) used to print "No deployed files +// tracked... may need to be redeployed" - false, and actively misleading a +// user debugging exactly the postsmoke bug. It must instead say the mod +// participates in the profile's merged pak. +func TestDoModFiles_DeployCompile_ExmodzModExplainsMergedPak(t *testing.T) { + configDir = t.TempDir() + dataDir = t.TempDir() + installDir := t.TempDir() + + svc, err := core.NewService(core.ServiceConfig{ConfigDir: configDir, DataDir: dataDir, CacheDir: t.TempDir()}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + game := &domain.Game{ + ID: "icarus", Name: "Icarus", InstallPath: installDir, ModPath: t.TempDir(), + DeployMode: domain.DeployCompile, LinkMethod: domain.LinkCopy, + SourceIDs: map[string]string{"fake-compiler": "external-icarus-id"}, + } + require.NoError(t, svc.AddGame(game)) + pm := getProfileManager(svc) + _, err = pm.Create(game.ID, "default") + require.NoError(t, err) + require.NoError(t, pm.SetDefault(game.ID, "default")) + + const modID, version, fileID = "bear-mount", "1.0", "exmodz-file" + gameCache := svc.GetGameCache(game) + require.NoError(t, gameCache.Store(game.ID, "fake-compiler", modID, version, cache.RetainedSourceName(fileID), []byte("bear-bytes"))) + require.NoError(t, svc.SaveInstalledMod(&domain.InstalledMod{ + Mod: domain.Mod{ID: modID, SourceID: "fake-compiler", Name: "Bear Mount", Version: version, GameID: game.ID}, + ProfileName: "default", + Enabled: true, + FileIDs: []string{fileID}, + UpdatePolicy: domain.UpdateNotify, + })) + require.NoError(t, pm.UpsertMod(game.ID, "default", domain.ModReference{SourceID: "fake-compiler", ModID: modID, Version: version, FileIDs: []string{fileID}})) + + oldSource, oldProfile := modSource, modProfile + modSource, modProfile = "fake-compiler", "default" + t.Cleanup(func() { modSource, modProfile = oldSource, oldProfile }) + + out := captureStdout(t, func() error { + return doModFiles(svc, game, modID) + }) + + assert.Contains(t, out, "merged pak") + assert.NotContains(t, out, "No deployed files tracked", "the old, false message must not survive alongside the new one") + assert.NotContains(t, out, "may need to be redeployed") +} + +// TestDoModFiles_NonCompile_ZeroFiles_KeepsOriginalMessage guards against +// over-broadening the #197 UX fix: a mod with genuinely zero tracked files +// for a reason OTHER THAN "it's a validated exmodz entry" (e.g. a plain +// DeployLink game with a stale/broken record) must keep the original +// "may need to be redeployed" guidance, not the merged-pak explanation. +func TestDoModFiles_NonCompile_ZeroFiles_KeepsOriginalMessage(t *testing.T) { + configDir = t.TempDir() + dataDir = t.TempDir() + + svc, err := core.NewService(core.ServiceConfig{ConfigDir: configDir, DataDir: dataDir, CacheDir: t.TempDir()}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + game := &domain.Game{ + ID: "other-game", Name: "Other Game", InstallPath: t.TempDir(), ModPath: t.TempDir(), + DeployMode: domain.DeployExtract, LinkMethod: domain.LinkCopy, + SourceIDs: map[string]string{"fake-source": "external-other-id"}, + } + require.NoError(t, svc.AddGame(game)) + pm := getProfileManager(svc) + _, err = pm.Create(game.ID, "default") + require.NoError(t, err) + require.NoError(t, pm.SetDefault(game.ID, "default")) + + const modID, version = "broken-mod", "1.0" + require.NoError(t, svc.SaveInstalledMod(&domain.InstalledMod{ + Mod: domain.Mod{ID: modID, SourceID: "fake-source", Name: "Broken Mod", Version: version, GameID: game.ID}, + ProfileName: "default", + Enabled: true, + UpdatePolicy: domain.UpdateNotify, + })) + require.NoError(t, pm.UpsertMod(game.ID, "default", domain.ModReference{SourceID: "fake-source", ModID: modID, Version: version})) + + oldSource, oldProfile := modSource, modProfile + modSource, modProfile = "fake-source", "default" + t.Cleanup(func() { modSource, modProfile = oldSource, oldProfile }) + + out := captureStdout(t, func() error { + return doModFiles(svc, game, modID) + }) + + assert.Contains(t, out, "No deployed files tracked") + assert.Contains(t, out, "may need to be redeployed") + assert.NotContains(t, out, "merged pak") +} diff --git a/cmd/lmm/verify.go b/cmd/lmm/verify.go index c030692..86fb06e 100644 --- a/cmd/lmm/verify.go +++ b/cmd/lmm/verify.go @@ -81,9 +81,10 @@ pak (#196, "the Friday problem" - a weekly base pak refresh silently reverts a compiled mod's patched tables, with nothing to notice otherwise): - ? NAME - RECOMPILE NEEDED the game's base pak has changed - since this mod was compiled; run - 'lmm update' to recompile it + ? NAME - RECOMPILE NEEDED the merged pak's inputs changed + (base pak update, or missing from + the game directory); run 'lmm + update --all' to fix it This check is entirely local (no source contacted) and applies to every compiled mod regardless of source, including local imports. --fix does @@ -357,9 +358,15 @@ func doVerify(cmd *cobra.Command, svc *core.Service, game *domain.Game, args []s checked++ if staleUpd != nil { if jsonOutput { - jsonFiles = append(jsonFiles, verifyFileJSON{ModID: staleUpd.InstalledMod.ID, ModName: staleUpd.InstalledMod.Name, Status: "stale_compile"}) + jsonFiles = append(jsonFiles, verifyFileJSON{ModID: staleUpd.InstalledMod.ID, ModName: staleUpd.InstalledMod.Name, Status: "stale_compile", Note: staleUpd.RecompileReason}) } else { - fmt.Printf("%s %s - RECOMPILE NEEDED (base pak updated - run 'lmm update' to fix)\n", colorYellow("?"), staleUpd.InstalledMod.Name) + // #197 postsmoke UX fix: use the real reason + // (RecompileReason distinguishes a fingerprint mismatch + // from a missing artifact) and name the flag that actually + // applies it - this row is always notify-policy (the + // synthetic merged-pak mod's zero-value UpdatePolicy), so + // bare 'lmm update' would only report it again, not fix it. + fmt.Printf("%s %s - RECOMPILE NEEDED (%s - run 'lmm update --all' to fix)\n", colorYellow("?"), staleUpd.InstalledMod.Name, staleUpd.RecompileReason) } warnings++ } diff --git a/cmd/lmm/verify_recompile_test.go b/cmd/lmm/verify_recompile_test.go index 9a44835..734a97a 100644 --- a/cmd/lmm/verify_recompile_test.go +++ b/cmd/lmm/verify_recompile_test.go @@ -40,6 +40,38 @@ func TestDoVerify_StaleCompile_ReportedAsWarning(t *testing.T) { output := buf.String() assert.Contains(t, output, "RECOMPILE NEEDED") assert.Contains(t, output, "Bear Mount") + assert.Contains(t, output, "base pak updated", "this fixture's staleness is a fingerprint mismatch, not a missing artifact") + assert.Contains(t, output, "lmm update --all", "notify-policy rows aren't applied by bare 'lmm update' - the hint must name the flag that actually applies them") +} + +// TestDoVerify_StaleCompile_NotDeployed_ReasonSaysSo is the #197 postsmoke +// UX regression test: the "RECOMPILE NEEDED" hint always said "base pak +// updated", even when the merged pak's fingerprint still matched and the +// real problem was a missing deployed artifact (the #197 I5 wedge case). +// CheckMergedPakStaleness now distinguishes the two - this proves doVerify +// actually surfaces the real reason instead of the fingerprint-mismatch +// text unconditionally. +func TestDoVerify_StaleCompile_NotDeployed_ReasonSaysSo(t *testing.T) { + svc, game, _, _ := setupDoUpdateRecompileTest(t) + require.NoError(t, svc.SaveFileChecksum("fake-compiler", "bear-mount", game.ID, "default", "exmodz-file-id", "deadbeef")) + _, err := svc.SyncMergedPak(context.Background(), game, "default") + require.NoError(t, err) + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + require.NoError(t, os.Remove(deployedPath)) + + verifyProfile = "default" + t.Cleanup(func() { verifyProfile = "" }) + + cmd := &cobra.Command{} + cmd.SetContext(context.Background()) + + out := captureStdout(t, func() error { + return doVerify(cmd, svc, game, nil) + }) + + assert.Contains(t, out, "RECOMPILE NEEDED") + assert.Contains(t, out, "not deployed") + assert.NotContains(t, out, "base pak updated", "the fingerprint still matches - blaming the base pak here is false") } // TestDoVerify_StaleCompile_JSON is the --json sibling of the above. @@ -77,6 +109,7 @@ func TestDoVerify_StaleCompile_JSON(t *testing.T) { // #197: a merged-pak staleness row's identity is the SYNTHETIC // merged-pak mod, not the contributing "bear-mount" mod. assert.Equal(t, "merged-pak", found.ModID) + assert.Equal(t, "base pak updated", found.Note, "the JSON note should carry the same real reason as the text-mode hint") assert.GreaterOrEqual(t, out.Warnings, 1) } diff --git a/docs/man/man1/lmm-verify.1 b/docs/man/man1/lmm-verify.1 index 56314b5..0cbe8d1 100644 --- a/docs/man/man1/lmm-verify.1 +++ b/docs/man/man1/lmm-verify.1 @@ -60,9 +60,10 @@ reverts a compiled mod's patched tables, with nothing to notice otherwise): .EX -? NAME - RECOMPILE NEEDED the game's base pak has changed - since this mod was compiled; run - 'lmm update' to recompile it +? NAME - RECOMPILE NEEDED the merged pak's inputs changed + (base pak update, or missing from + the game directory); run 'lmm + update --all' to fix it .EE .PP diff --git a/internal/core/merged_pak.go b/internal/core/merged_pak.go index 70c4a32..3c530f8 100644 --- a/internal/core/merged_pak.go +++ b/internal/core/merged_pak.go @@ -355,6 +355,12 @@ func (s *Service) CheckMergedPakStaleness(game *domain.Game, profileName string) gameCache := s.GetGameCache(game) cachePath := gameCache.ModPath(game.ID, domain.SourceMerged, mergedPakModID, mergedPakVersion) stored, ok := readMergedFingerprint(cachePath) + // #197 postsmoke UX fix: reason defaults to the fingerprint-mismatch + // case ("base pak updated") and is only downgraded to "not deployed" + // below, once we know the fingerprint actually matched - callers + // (verify/update's console output) must not blame the base pak when + // the real cause is a missing artifact. + reason := "base pak updated" if ok { if eq, eqErr := mergedFingerprintsEqual(current, stored); eqErr == nil && eq { // #197 I5 fix: mirrors syncMergedPak's identical fast-path @@ -367,6 +373,7 @@ func (s *Service) CheckMergedPakStaleness(game *domain.Game, profileName string) if _, statErr := os.Stat(filepath.Join(game.ModPath, mergedPakFileName)); statErr == nil { return nil, nil } + reason = "not deployed" } } @@ -379,6 +386,7 @@ func (s *Service) CheckMergedPakStaleness(game *domain.Game, profileName string) }, NewVersion: mergedPakVersion, RecompileNeeded: true, + RecompileReason: reason, }, nil } diff --git a/internal/core/merged_pak_staleness_test.go b/internal/core/merged_pak_staleness_test.go index 6e6bd82..85d67ed 100644 --- a/internal/core/merged_pak_staleness_test.go +++ b/internal/core/merged_pak_staleness_test.go @@ -34,6 +34,7 @@ func TestCheckMergedPakStaleness_StaleAfterModEnable(t *testing.T) { require.NotNil(t, upd) require.True(t, upd.RecompileNeeded) require.Equal(t, upd.InstalledMod.Version, upd.NewVersion, "a staleness row has no real version change") + require.Equal(t, "base pak updated", upd.RecompileReason, "inputs changed (a mod was enabled) - the reason must say so, not the not-deployed case") } func TestCheckMergedPakStaleness_NilWhenNoMergedPakEverGenerated(t *testing.T) { @@ -154,4 +155,5 @@ func TestCheckMergedPakStaleness_MissingArtifact_ReportsStale(t *testing.T) { require.NoError(t, err) require.NotNil(t, upd, "a missing deployed artifact must be reported stale even though the fingerprint hasn't changed") require.True(t, upd.RecompileNeeded) + require.Equal(t, "not deployed", upd.RecompileReason, "the fingerprint still matches - the real reason is the missing artifact, not a base pak change") } diff --git a/internal/domain/mod.go b/internal/domain/mod.go index 33b8ec9..39d6ca3 100644 --- a/internal/domain/mod.go +++ b/internal/domain/mod.go @@ -133,6 +133,13 @@ type Update struct { // changed, only the base pak has - so callers must not treat NewVersion // as a real version bump when this is set. RecompileNeeded bool + // RecompileReason qualifies RecompileNeeded with why a recompile/resync + // is needed (#197 postsmoke UX fix): "base pak updated" when the + // merged/compiled fingerprint no longer matches current inputs, "not + // deployed" when the fingerprint still matches but the artifact is + // missing from the game directory (#197 I5's wedge case). Empty when + // RecompileNeeded is false. + RecompileReason string } // ModKey returns a unique lookup key for a mod: "sourceID:modID". From b7c025435395bba8a7f810598a8a858ad31a0b3c Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sat, 1 Aug 2026 23:25:41 -0400 Subject: [PATCH 86/96] fix: thread merged-pak sync outcome into install messages; fix --fix docs (#200 review) Two Copilot review findings on PR #200: 1. install.go:657 (+732, 1213) - the "Installed (merged pak updated)" success line printed unconditionally for a DeployCompile zero-file mod, even when the sync that actually updates the merged pak is non-fatal and had just failed (with a Warning already on stderr) - stdout and stderr told contradictory stories. - doInstall (STRICT path): InstallResult gains MergedPakSyncFailed, set by ApplyInstall's own end-of-call sync attempt; the success line threads it directly, since the sync has already happened by the time it prints. - doInstallBatch (dependency/BATCH path) and batchInstallMods (multi-select path): the sync is deferred to ONCE per batch, AFTER every per-mod "Installed" line would otherwise print live - so the outcome literally isn't knowable yet at that point. Both now buffer the affected mods' names as they're installed and print their completion lines only after the batch's one sync attempt actually runs, positioned after its stderr Warning ("...see warning above" is now literally true, not just illustrative). 2. verify.go:663 - doVerify --fix has synced the merged pak since the prior #197 fix wave, but the command's --help text (and generated man page) still claimed "--fix does not repair" compile/merged-pak staleness. Corrected to describe what --fix actually does; `make man` regenerated docs/man/man1/lmm-verify.1 (enforced by TestGenManTree_MatchesCommittedPages). RED->GREEN verified for all three install.go sites (temporarily reverted each fix, confirmed the false-success-line symptom, restored, confirmed pass) plus a fixture fix (TestDoInstall_DeployCompile_AnnouncesRetaining was missing svc.AddGame, silently sync-failing and printing the wrong branch once the wording was made accurate). Co-Authored-By: Claude Sonnet 5 --- cmd/lmm/install.go | 68 +++++++++++++-- cmd/lmm/install_compile_test.go | 149 +++++++++++++++++++++++++++++++- cmd/lmm/verify.go | 5 +- docs/man/man1/lmm-verify.1 | 5 +- internal/core/flows.go | 11 +++ 5 files changed, 223 insertions(+), 15 deletions(-) diff --git a/cmd/lmm/install.go b/cmd/lmm/install.go index 556da74..847a904 100644 --- a/cmd/lmm/install.go +++ b/cmd/lmm/install.go @@ -649,11 +649,17 @@ func doInstall(ctx context.Context, service *core.Service, game *domain.Game, ar // files of its own by design (validate+retain only - it participates // in the profile's shared merged pak instead, synced separately // above) - "Files deployed: 0" read as a failure, not the correct, - // expected outcome it actually is. - if game.DeployMode == domain.DeployCompile && result.FilesDeployed == 0 { - fmt.Println(" Installed (merged pak updated)") - } else { + // expected outcome it actually is. Copilot review (#200): that sync is + // non-fatal, so this line unconditionally claimed "merged pak updated" + // even when it had just failed loudly on stderr above - thread the + // actual outcome instead of asserting success either way. + switch { + case game.DeployMode != domain.DeployCompile || result.FilesDeployed != 0: fmt.Printf(" Files deployed: %d\n", result.FilesDeployed) + case result.MergedPakSyncFailed: + fmt.Println(" Installed; merged pak sync FAILED — see warning above") + default: + fmt.Println(" Installed (merged pak updated)") } fmt.Printf(" Added to profile: %s\n", profileName) @@ -697,6 +703,15 @@ func doInstallBatch(ctx context.Context, service *core.Service, game *domain.Gam Force: installForce, } + // pendingCompileDeps buffers the display names of DeployCompile + // zero-file dependencies as InstallDepInstalled events arrive live - + // their "merged pak updated" claim can't be verified until the SINGLE + // end-of-batch sync (inside ApplyInstall) actually runs, which happens + // after every per-dep event has already streamed. Printed once + // ApplyInstall returns with the real outcome (#197 postsmoke Copilot + // review fix, #200). + var pendingCompileDeps []string + // progress prints every diagnostic and status line at its exact point // of occurrence, driven entirely by core.ApplyInstall's BATCH-path // progress events - reproducing batchInstallMods' console output @@ -731,9 +746,13 @@ func doInstallBatch(ctx context.Context, service *core.Service, game *domain.Gam case core.InstallDepInstalled: // #197 postsmoke UX fix: see the single-mod path's identical // fix above - a DeployCompile ".exmodz" dependency deploys - // zero files of its own by design. + // zero files of its own by design. The "merged pak updated" + // half of that claim can't be printed yet (see + // pendingCompileDeps above) - deferred until ApplyInstall + // returns and the batch's one sync attempt is known to have + // succeeded or failed. if game.DeployMode == domain.DeployCompile && p.FilesExtracted == 0 { - fmt.Println(" ✓ Installed (merged pak updated)") + pendingCompileDeps = append(pendingCompileDeps, p.ModName) } else { fmt.Printf(" ✓ Installed (%d files)\n", p.FilesExtracted) } @@ -762,6 +781,17 @@ func doInstallBatch(ctx context.Context, service *core.Service, game *domain.Gam return err } + // The batch's one merged-pak sync attempt (inside ApplyInstall) has + // now happened - print the deferred per-dependency completion lines + // pendingCompileDeps buffered above, with the outcome finally known. + for _, name := range pendingCompileDeps { + if result.MergedPakSyncFailed { + fmt.Printf(" ✓ %s: installed; merged pak sync FAILED — see warning above\n", name) + } else { + fmt.Printf(" ✓ %s: Installed (merged pak updated)\n", name) + } + } + fmt.Printf("\n--- Summary ---\n") fmt.Printf("Installed: %d\n", len(result.Installed)) if len(result.Failed) > 0 { @@ -1060,6 +1090,11 @@ func batchInstallMods(ctx context.Context, service *core.Service, game *domain.G var installed, failed []string var hookErrors []error + // pendingCompileMods buffers the display names of DeployCompile + // zero-file mods as they're installed - their "merged pak updated" + // claim can't be verified until the batch's one sync attempt, below + // the loop, actually runs (#197 postsmoke Copilot review fix, #200). + var pendingCompileMods []string for i, mod := range mods { fmt.Printf("\n[%d/%d] Installing: %s v%s\n", i+1, len(mods), mod.Name, mod.Version) @@ -1213,9 +1248,12 @@ func batchInstallMods(ctx context.Context, service *core.Service, game *domain.G // #197 postsmoke UX fix: see doInstall's identical fix - a // DeployCompile ".exmodz" mod deploys zero files of its own by // design (validate+retain only; the merged pak sync below is what - // actually deploys it). + // actually deploys it). The "merged pak updated" half of that + // claim can't be printed yet (see pendingCompileMods above) - + // deferred until the sync below actually runs and its outcome is + // known. if game.DeployMode == domain.DeployCompile && downloadResult.FilesExtracted == 0 { - fmt.Println(" ✓ Installed (merged pak updated)") + pendingCompileMods = append(pendingCompileMods, mod.Name) } else { fmt.Printf(" ✓ Installed (%d files)\n", downloadResult.FilesExtracted) } @@ -1246,7 +1284,8 @@ func batchInstallMods(ctx context.Context, service *core.Service, game *domain.G // warn the user. Sync failures are printed unconditionally (not // --verbose-gated): if this had failed loudly the first time, the user // would have noticed immediately instead of silently missing content. - if syncWarnings, syncErr := service.SyncMergedPak(ctx, game, profileName); syncErr != nil { + syncWarnings, syncErr := service.SyncMergedPak(ctx, game, profileName) + if syncErr != nil { fmt.Fprintf(os.Stderr, "Warning: could not sync merged pak: %v\n", syncErr) } else { for _, w := range syncWarnings { @@ -1254,6 +1293,17 @@ func batchInstallMods(ctx context.Context, service *core.Service, game *domain.G } } + // The sync above has now happened - print the deferred per-mod + // completion lines pendingCompileMods buffered during the loop, with + // the outcome finally known (#197 postsmoke Copilot review fix, #200). + for _, name := range pendingCompileMods { + if syncErr != nil { + fmt.Printf(" ✓ %s: installed; merged pak sync FAILED — see warning above\n", name) + } else { + fmt.Printf(" ✓ %s: Installed (merged pak updated)\n", name) + } + } + // Summary fmt.Printf("\n--- Summary ---\n") fmt.Printf("Installed: %d\n", len(installed)) diff --git a/cmd/lmm/install_compile_test.go b/cmd/lmm/install_compile_test.go index e7e5200..eb7fcad 100644 --- a/cmd/lmm/install_compile_test.go +++ b/cmd/lmm/install_compile_test.go @@ -79,6 +79,11 @@ func TestDoInstall_DeployCompile_AnnouncesRetaining(t *testing.T) { svc, game, src := setupDoInstallTest(t) game.DeployMode = domain.DeployCompile game.InstallPath = t.TempDir() + // SyncMergedPak resolves the game's configured sources, which requires + // the game to be registered - setupDoInstallTest's bare *domain.Game + // construction skips this (a shared fixture used by many non-compile + // tests too); the production CLI always has this via withGameService. + require.NoError(t, svc.AddGame(game)) basePak := filepath.Join(game.InstallPath, "Icarus", "Content", "Data", "data.pak") require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) @@ -155,6 +160,52 @@ func TestBatchInstallMods_DeployCompile_DeploysMergedPak(t *testing.T) { "#197 postsmoke UX fix: each zero-file exmodz mod must say what happened, not print the misleading '(0 files)'") } +// TestBatchInstallMods_DeployCompile_SyncFailure_LinesDontClaimSuccess is +// the #197 postsmoke Copilot review fix (#200): batchInstallMods' per-mod +// "✓ Installed (merged pak updated)" line used to print unconditionally, +// even when the batch's own sync attempt (right below the loop) had just +// failed and printed a Warning to stderr - a stdout/stderr contradiction. +// Proves both installed mods' completion lines instead say the sync +// FAILED, printed after the stderr warning. +func TestBatchInstallMods_DeployCompile_SyncFailure_LinesDontClaimSuccess(t *testing.T) { + svc, game, src := setupDoInstallTest(t) + game.DeployMode = domain.DeployCompile + game.InstallPath = t.TempDir() + + basePak := filepath.Join(game.InstallPath, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + writeFakeBasePak(t, basePak) + + compiler := &compilerInstallSource{fakeInstallSource: src, mergeErr: assert.AnError} + svc.RegisterSource(compiler) + require.NoError(t, svc.AddGame(game)) + + bearMod := &domain.Mod{ID: "bear-mount", SourceID: "test-src", Name: "Bear Mount", Version: "1.0", GameID: "g1"} + wolfMod := &domain.Mod{ID: "wolf-mount", SourceID: "test-src", Name: "Wolf Mount", Version: "1.0", GameID: "g1"} + src.AddMod(bearMod, []domain.DownloadableFile{{ID: "bear-exmodz", Name: "Bear Mount", FileName: "Bear_Mount.exmodz", IsPrimary: true, Category: "MAIN"}}) + src.AddMod(wolfMod, []domain.DownloadableFile{{ID: "wolf-exmodz", Name: "Wolf Mount", FileName: "Wolf_Mount.exmodz", IsPrimary: true, Category: "MAIN"}}) + src.AddDownload("bear-exmodz", []byte("bear-bytes")) + src.AddDownload("wolf-exmodz", []byte("wolf-bytes")) + + oldStderr := os.Stderr + r, w, pipeErr := os.Pipe() + require.NoError(t, pipeErr) + os.Stderr = w + out, err := captureStdoutErr(t, func() error { + return batchInstallMods(context.Background(), svc, game, []*domain.Mod{bearMod, wolfMod}, "default") + }) + _ = w.Close() + os.Stderr = oldStderr + require.NoError(t, err, "a merge failure is non-fatal to the batch install itself") + + var stderrBuf bytes.Buffer + _, _ = stderrBuf.ReadFrom(r) + assert.Contains(t, stderrBuf.String(), "Warning:") + + assert.Equal(t, 2, strings.Count(out, "installed; merged pak sync FAILED — see warning above")) + assert.NotContains(t, out, "Installed (merged pak updated)") +} + // TestDoInstall_DeployCompile_SyncFailure_PrintsLoudly is the #197 // postsmoke "must be LOUD" regression test: ApplyInstall's own sync call // used to only append to result.Warnings, which doInstall (the single-mod @@ -183,12 +234,106 @@ func TestDoInstall_DeployCompile_SyncFailure_PrintsLoudly(t *testing.T) { r, w, pipeErr := os.Pipe() require.NoError(t, pipeErr) os.Stderr = w - err := doInstall(context.Background(), svc, game, nil) + out := captureStdout(t, func() error { + return doInstall(context.Background(), svc, game, nil) + }) _ = w.Close() os.Stderr = oldStderr - require.NoError(t, err, "a merge failure is non-fatal to the install itself - the mod is still validated+retained+recorded") var buf bytes.Buffer _, _ = buf.ReadFrom(r) assert.Contains(t, buf.String(), "Warning:", "a merge failure during install must print a Warning unconditionally, not silently vanish into a discarded result") + + // #197 postsmoke Copilot review fix (#200): the success line must not + // claim "merged pak updated" when the sync above it just failed - it + // contradicted the loud Warning on stderr. + assert.Contains(t, out, "merged pak sync FAILED — see warning above") + assert.NotContains(t, out, "Installed (merged pak updated)") +} + +// TestDoInstallBatch_DeployCompile_DeploysMergedPak drives the dependency +// (BATCH) path - doInstall -> doInstallBatch -> core.ApplyInstall - with two +// exmodz mods (a dependency and its primary), proving this path (distinct +// from batchInstallMods, the multi-select search path) also deploys the +// merged pak and reports each mod correctly. +func TestDoInstallBatch_DeployCompile_DeploysMergedPak(t *testing.T) { + svc, game, src := setupDoInstallTest(t) + game.DeployMode = domain.DeployCompile + game.InstallPath = t.TempDir() + require.NoError(t, svc.AddGame(game)) + installYes = true + + basePak := filepath.Join(game.InstallPath, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + writeFakeBasePak(t, basePak) + + compiler := &compilerInstallSource{fakeInstallSource: src} + svc.RegisterSource(compiler) + + dep := &domain.Mod{ID: "dep1", SourceID: "test-src", Name: "Wolf Mount", Version: "1.0", GameID: "g1"} + root := &domain.Mod{ID: "mod1", SourceID: "test-src", Name: "Bear Mount", Version: "1.0", GameID: "g1", + Dependencies: []domain.ModReference{{SourceID: "test-src", ModID: "dep1"}}} + src.AddMod(dep, []domain.DownloadableFile{{ID: "dep-file", FileName: "Wolf_Mount.exmodz", IsPrimary: true}}) + src.AddDownload("dep-file", []byte("wolf-bytes")) + src.AddMod(root, []domain.DownloadableFile{{ID: "main", FileName: "Bear_Mount.exmodz", IsPrimary: true}}) + src.AddDownload("main", []byte("bear-bytes")) + + out := captureStdout(t, func() error { + return doInstall(context.Background(), svc, game, nil) + }) + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + data, readErr := os.ReadFile(deployedPath) + require.NoError(t, readErr, "doInstallBatch must sync the merged pak - both mods deploy zero files of their own") + assert.Contains(t, string(data), "wolf-bytes") + assert.Contains(t, string(data), "bear-bytes") + + assert.Equal(t, 2, strings.Count(out, "Installed (merged pak updated)"), + "#197 postsmoke UX fix: each zero-file exmodz dependency/primary must say what happened") +} + +// TestDoInstallBatch_DeployCompile_SyncFailure_LinesDontClaimSuccess is the +// #197 postsmoke Copilot review fix (#200) for the dependency (BATCH) path: +// each per-dependency "✓ Installed (merged pak updated)" line streams live, +// BEFORE ApplyInstall's own end-of-batch sync attempt runs - so the claim +// can't be verified at print time. Proves the completion lines are deferred +// until the real outcome is known and say the sync FAILED instead. +func TestDoInstallBatch_DeployCompile_SyncFailure_LinesDontClaimSuccess(t *testing.T) { + svc, game, src := setupDoInstallTest(t) + game.DeployMode = domain.DeployCompile + game.InstallPath = t.TempDir() + require.NoError(t, svc.AddGame(game)) + installYes = true + + basePak := filepath.Join(game.InstallPath, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + writeFakeBasePak(t, basePak) + + compiler := &compilerInstallSource{fakeInstallSource: src, mergeErr: assert.AnError} + svc.RegisterSource(compiler) + + dep := &domain.Mod{ID: "dep1", SourceID: "test-src", Name: "Wolf Mount", Version: "1.0", GameID: "g1"} + root := &domain.Mod{ID: "mod1", SourceID: "test-src", Name: "Bear Mount", Version: "1.0", GameID: "g1", + Dependencies: []domain.ModReference{{SourceID: "test-src", ModID: "dep1"}}} + src.AddMod(dep, []domain.DownloadableFile{{ID: "dep-file", FileName: "Wolf_Mount.exmodz", IsPrimary: true}}) + src.AddDownload("dep-file", []byte("wolf-bytes")) + src.AddMod(root, []domain.DownloadableFile{{ID: "main", FileName: "Bear_Mount.exmodz", IsPrimary: true}}) + src.AddDownload("main", []byte("bear-bytes")) + + oldStderr := os.Stderr + r, w, pipeErr := os.Pipe() + require.NoError(t, pipeErr) + os.Stderr = w + out := captureStdout(t, func() error { + return doInstall(context.Background(), svc, game, nil) + }) + _ = w.Close() + os.Stderr = oldStderr + + var stderrBuf bytes.Buffer + _, _ = stderrBuf.ReadFrom(r) + assert.Contains(t, stderrBuf.String(), "Warning:") + + assert.Equal(t, 2, strings.Count(out, "installed; merged pak sync FAILED — see warning above")) + assert.NotContains(t, out, "Installed (merged pak updated)") } diff --git a/cmd/lmm/verify.go b/cmd/lmm/verify.go index 86fb06e..d1cf21b 100644 --- a/cmd/lmm/verify.go +++ b/cmd/lmm/verify.go @@ -87,8 +87,9 @@ otherwise): update --all' to fix it This check is entirely local (no source contacted) and applies to every -compiled mod regardless of source, including local imports. --fix does -not repair it - use 'lmm update' (or 'lmm update --all'). +compiled mod regardless of source, including local imports. Use --fix to +repair it: it resyncs the profile's merged pak (recompiling and +redeploying it if needed), the same repair 'lmm update --all' applies. Mods installed from a local source, mods requiring manual download, and mods with no recorded file IDs are skipped silently - there is nothing diff --git a/docs/man/man1/lmm-verify.1 b/docs/man/man1/lmm-verify.1 index 0cbe8d1..2b065e4 100644 --- a/docs/man/man1/lmm-verify.1 +++ b/docs/man/man1/lmm-verify.1 @@ -68,8 +68,9 @@ otherwise): .PP This check is entirely local (no source contacted) and applies to every -compiled mod regardless of source, including local imports. --fix does -not repair it - use 'lmm update' (or 'lmm update --all'). +compiled mod regardless of source, including local imports. Use --fix to +repair it: it resyncs the profile's merged pak (recompiling and +redeploying it if needed), the same repair 'lmm update --all' applies. .PP Mods installed from a local source, mods requiring manual download, and diff --git a/internal/core/flows.go b/internal/core/flows.go index 7138453..5797864 100644 --- a/internal/core/flows.go +++ b/internal/core/flows.go @@ -3249,6 +3249,16 @@ type InstallResult struct { // never printed a file count, only Installed/Failed - see Failed). FilesDeployed int + // MergedPakSyncFailed is true when this call's own end-of-install + // syncMergedPak attempt returned a hard error (#197 postsmoke review + // fix - Copilot flagged that a DeployCompile zero-file mod's success + // line unconditionally claimed "merged pak updated" even when the + // non-fatal sync failed, contradicting the loud Warning already on + // stderr). False when the sync succeeded, including when it returned + // its own non-fatal merge warnings - those still leave the pak + // deployed. Always false for a non-DeployCompile game. + MergedPakSyncFailed bool + Warnings []string Notes []string } @@ -3765,6 +3775,7 @@ func (s *Service) ApplyInstall(ctx context.Context, game *domain.Game, plan *Ins if syncWarnings, syncErr := s.syncMergedPak(ctx, game, plan.Profile); syncErr != nil { msg := fmt.Sprintf("syncing merged pak: %v", syncErr) result.Warnings = append(result.Warnings, msg) + result.MergedPakSyncFailed = true emit(DeployProgress{Phase: InstallWarning, Detail: msg}) } else { for _, w := range syncWarnings { From 00802cf9a0ff7a15e0c91323ce9ca0461cfaf2ba Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sun, 2 Aug 2026 00:22:37 -0400 Subject: [PATCH 87/96] feat: document merge precedence; display list in profile load order (#201) lmm list showed mods in DB install order (installed_at), which has no relationship to what actually decides merge precedence for a deploy_mode: compile game (Icarus). Switched to core.OrderByProfile - the same seam the TUI's mod list already uses (Overview, service_core.go) - rather than the deploy-only GetInstalledModsInProfileOrder: that helper deliberately OMITS a mod absent from the profile's load order (correct for deploy, where an untracked mod must never silently deploy), which would have made such a mod vanish from a listing instead of just showing up first (lowest priority, since it has no claim to "final say"). Using OrderByProfile gives list.go the exact same order the TUI already shows, so CLI and TUI now genuinely agree - true parity, confirmed with the coordinator - rather than inventing a third, list-only convention. Added a "Merge precedence" paragraph to README and docs/configuration.md, verified against the actual merge implementation (internal/source/icarus/merge.go): later-in-load-order mods win conflicting table-row fields via a per-field upsert (untouched fields from earlier mods survive); bundled assets are whole-file last-wins with a warning (install/update surface it; reorder itself regenerates the pak silently); the bottom of the load order has final say; lmm profile reorder regenerates the merged pak immediately. list's help text now names the load-order behavior explicitly (ran `make man` to regenerate the stale committed man page the genman test caught). --- CHANGELOG.md | 1 + README.md | 2 + cmd/lmm/list.go | 20 ++++++++ cmd/lmm/list_order_test.go | 97 ++++++++++++++++++++++++++++++++++++++ cmd/lmm/list_test.go | 11 +++++ docs/configuration.md | 2 + docs/man/man1/lmm-list.1 | 7 +++ 7 files changed, 140 insertions(+) create mode 100644 cmd/lmm/list_order_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index f95c851..75b0588 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- `lmm list` now displays mods in the profile's load order (the same order `lmm profile reorder` sets and the TUI's mod list already showed) instead of DB install order (`installed_at`) — the visible order is now the order that actually decides merge precedence for a `deploy_mode: compile` game. A mod installed but missing from the load order is still shown, never silently dropped, placed first (lowest priority). README and `docs/configuration.md` gain a "Merge precedence" paragraph explaining that later-in-load-order mods win conflicting table-row _fields_ (a per-field upsert; untouched fields from earlier mods survive) while bundled assets are whole-file last-wins with a warning, and that `lmm profile reorder` regenerates the merged pak immediately (#201) - An unrecognized, non-empty `link_method` (`games.yaml`, profile files, imported profiles) or `deploy_mode` (`games.yaml`; also `lmm game detect`'s `steam-games.yaml`) is now a load-time error naming the field, the offending value, the owning game/profile, and the valid options — instead of silently falling back to the default (`symlink`/`extract`). **Breaking for configs that were already silently misbehaving:** a typo like `deploy_mode: compil` previously ran as `extract` with no warning; it now refuses to load until fixed. An empty/absent value is unaffected and keeps today's default exactly (#172) ### Fixed diff --git a/README.md b/README.md index 0b6d9b1..cd6d3e5 100644 --- a/README.md +++ b/README.md @@ -433,6 +433,8 @@ games: Steam auto-detection (`lmm game detect`) knows about Icarus (App ID `1149460`) and generates an equivalent entry for you, `install_path`/`mod_path` filled in from your actual Steam library — the YAML above is kept here as reference for what gets written, not something you need to type by hand. +**Merge precedence**: with more than one `compile`-mode mod installed (currently Icarus only), the profile's load order — the same order `lmm list` displays and `lmm profile reorder` changes — decides how conflicting changes resolve. Mods are merged in load order, so a mod later in the list is applied later and wins conflicting _fields_ on a shared data-table row; it's a per-field upsert, not a whole-row overwrite, so untouched fields from earlier mods still survive. Bundled asset files can't compose that way — a same-path collision between two mods is whole-file last-wins, and installing or updating a colliding mod prints a warning naming both. Either way, the bottom of the load order has final say, and `lmm profile reorder` regenerates the merged pak immediately, so a reorder's effect on precedence is visible right away rather than at the next deploy. + ### Deployment Methods Mods can be deployed using three methods: diff --git a/cmd/lmm/list.go b/cmd/lmm/list.go index 703d981..faa28e8 100644 --- a/cmd/lmm/list.go +++ b/cmd/lmm/list.go @@ -48,6 +48,12 @@ var listCmd = &cobra.Command{ Short: "List installed mods", Long: `List all mods installed in the specified game and profile. +Mods are printed in the profile's load order (see 'lmm profile reorder') +- the same order that decides merge precedence for a compiled/merged pak: +a mod later in the load order is merged later and wins conflicting rows. +A mod installed but missing from the load order is still shown (never +silently dropped), placed first since it has no claim to the final say. + Use --profiles to list profile names for the game instead of mods. Examples: @@ -102,6 +108,20 @@ func doList(cmd *cobra.Command, service *core.Service, game *domain.Game) error } } + // #201: display the profile's load order - the order that actually + // decides merge precedence (later = merged later = wins) - not + // installed_at (GetInstalledMods' own DB order), which has no + // relationship to it. core.OrderByProfile, not the deploy-only + // GetInstalledModsInProfileOrder seam: that one deliberately OMITS a + // mod absent from the profile's load order (correct for deploy - an + // untracked mod must never silently deploy), which would make such a + // mod vanish from a listing instead of just being placed first (lowest + // priority, since it has no claim to "final say"). OrderByProfile is + // the same never-omitting seam the TUI's mod list already uses + // (internal/tui/service_core.go's Overview) - reusing it here keeps the + // CLI and TUI in agreement on what "the load order" looks like. + mods = core.OrderByProfile(profileYAML, mods) + if jsonOutput { out := listJSONOutput{GameID: game.ID, Profile: profileName, Mods: make([]listModJSON, len(mods))} for i, mod := range mods { diff --git a/cmd/lmm/list_order_test.go b/cmd/lmm/list_order_test.go new file mode 100644 index 0000000..a338ee8 --- /dev/null +++ b/cmd/lmm/list_order_test.go @@ -0,0 +1,97 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// modOrder returns each named mod's line index in out, in the order the +// names first appear - used to assert relative ordering without depending +// on exact column widths. +func modOrder(t *testing.T, out string, names ...string) []int { + t.Helper() + lines := strings.Split(out, "\n") + indices := make([]int, len(names)) + for i, name := range names { + indices[i] = -1 + for lineIdx, l := range lines { + if strings.Contains(l, name) { + indices[i] = lineIdx + break + } + } + require.NotEqual(t, -1, indices[i], "expected to find %q in output:\n%s", name, out) + } + return indices +} + +// TestList_DisplaysProfileLoadOrder_NotInstallOrder guards #201: `lmm list` +// used to print mods in DB install order (installed_at), not the profile's +// load order that actually decides merge precedence. Mod A is installed +// before Mod B (install order: A, B) but the profile's load order is then +// reversed to [B, A] - the listing must follow the load order, not +// installed_at. +func TestList_DisplaysProfileLoadOrder_NotInstallOrder(t *testing.T) { + svc, game := setupDoDeployTest(t) + seedDeployableMod(t, svc, game, "a", "Mod A", "a.esp") + seedDeployableMod(t, svc, game, "b", "Mod B", "b.esp") + + require.NoError(t, svc.NewProfileManager().ReorderMods(game.ID, "default", []domain.ModReference{ + {SourceID: "src", ModID: "b", Version: "1.0"}, + {SourceID: "src", ModID: "a", Version: "1.0"}, + })) + + t.Run("non-verbose", func(t *testing.T) { + out := listNonVerbose(t, svc, game) + idx := modOrder(t, out, "Mod B", "Mod A") + assert.Less(t, idx[0], idx[1], "Mod B (later in load order) must print before Mod A") + }) + + t.Run("verbose", func(t *testing.T) { + out := listVerbose(t, svc, game, false) + idx := modOrder(t, out, "Mod B", "Mod A") + assert.Less(t, idx[0], idx[1], "Mod B (later in load order) must print before Mod A") + }) + + t.Run("json", func(t *testing.T) { + raw := listVerbose(t, svc, game, true) + var out listJSONOutput + require.NoError(t, json.Unmarshal([]byte(raw), &out)) + require.Len(t, out.Mods, 2) + assert.Equal(t, "b", out.Mods[0].ID) + assert.Equal(t, "a", out.Mods[1].ID) + }) +} + +// TestList_ModMissingFromLoadOrder_StillShown guards the never-omit +// requirement (#201): GetInstalledModsInProfileOrder (deploy's seam) +// deliberately OMITS a mod absent from the profile's load order - correct +// for deploy, since an untracked mod must never silently deploy, but wrong +// for a listing, where every installed mod must still be visible. list.go +// uses core.OrderByProfile instead (the same seam the TUI's mod list +// already uses - internal/tui/service_core.go's Overview), which never +// omits: a load-order-absent mod is placed first (lowest priority - it has +// no claim to "final say"), never dropped. +func TestList_ModMissingFromLoadOrder_StillShown(t *testing.T) { + svc, game := setupDoDeployTest(t) + seedDeployableMod(t, svc, game, "a", "Tracked Mod", "a.esp") + + // Install "b" without ever adding it to the profile's load order - + // simulates the edge case a normal add/remove flow shouldn't produce, + // but which must not make the mod vanish from `lmm list`. + require.NoError(t, svc.SaveInstalledMod(&domain.InstalledMod{ + Mod: domain.Mod{ID: "b", SourceID: "src", Name: "Untracked Mod", Version: "1.0", GameID: game.ID}, + ProfileName: "default", + UpdatePolicy: domain.UpdateNotify, + Enabled: true, + })) + + out := listNonVerbose(t, svc, game) + assert.Contains(t, out, "Untracked Mod", "a mod absent from the profile's load order must still be listed") + assert.Contains(t, out, "2 mod(s)", "both the tracked and untracked mod must count toward the total") +} diff --git a/cmd/lmm/list_test.go b/cmd/lmm/list_test.go index f7b206e..196aa36 100644 --- a/cmd/lmm/list_test.go +++ b/cmd/lmm/list_test.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "strings" "testing" "github.com/spf13/cobra" @@ -35,6 +36,16 @@ func TestListCmd_Structure(t *testing.T) { assert.NotNil(t, listCmd.Flags().Lookup("profiles")) } +// TestListCmd_DocMentionsLoadOrder guards #201: the help text must describe +// the mod ordering it actually shows (the profile's load order, which +// decides merge precedence) rather than staying silent about it or, worse, +// claiming the old install order. +func TestListCmd_DocMentionsLoadOrder(t *testing.T) { + assert.Contains(t, listCmd.Long, "load order") + assert.NotContains(t, strings.ToLower(listCmd.Long), "install order", + "list must not claim install order - it shows profile load order") +} + func TestStatusCmd_Structure(t *testing.T) { assert.Equal(t, "status", statusCmd.Use) assert.NotEmpty(t, statusCmd.Short) diff --git a/docs/configuration.md b/docs/configuration.md index 74748cb..d5f8369 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -61,6 +61,8 @@ The `deploy_mode` option controls how downloaded mod archives are handled: - **`copy`**: Archives are copied as-is to the mod path without extraction. Use for games that expect mod files to remain as archives (e.g., Minecraft `.jar` files, some Unity games). - **`compile`**: The downloaded file is compiled into a new artifact before caching (currently Icarus only: an `.exmodz` diff is applied to the game's base data tables to produce a deployable `_P.pak`). Only sources that implement compiling support this mode. The base data tables are read directly from the installed game's own `data.pak`, so a compile always matches the installed game version and needs no network access. +**Merge precedence**: with more than one `compile`-mode mod installed, the merge applies each mod's changes in the profile's load order (the `mods` list's order - see [Profile files](#profile-files) below, and the same order `lmm list` displays) against the same evolving base tables, so a mod later in the load order is applied later. Table-row conflicts compose at the _field_ level: an upsert, not a whole-row overwrite, so two mods patching different fields of the same row - or different rows entirely - both survive; only a genuine same-row-same-field write is last-wins, which is an expected outcome of ordinary upserts, not something that gets a warning. Bundled asset files can't compose that way - a same-path asset collision between two mods is necessarily whole-file last-wins, and is reported as a warning (installing or updating a colliding mod prints it). Either way, the mod at the bottom of the load order has final say, and reordering the profile (`lmm profile reorder`) regenerates the merged pak immediately, so the new precedence takes effect right away. + Example: ```yaml diff --git a/docs/man/man1/lmm-list.1 b/docs/man/man1/lmm-list.1 index aee9f87..735d1bb 100644 --- a/docs/man/man1/lmm-list.1 +++ b/docs/man/man1/lmm-list.1 @@ -12,6 +12,13 @@ lmm-list - List installed mods .SH DESCRIPTION List all mods installed in the specified game and profile. +.PP +Mods are printed in the profile's load order (see 'lmm profile reorder') +- the same order that decides merge precedence for a compiled/merged pak: +a mod later in the load order is merged later and wins conflicting rows. +A mod installed but missing from the load order is still shown (never +silently dropped), placed first since it has no claim to the final say. + .PP Use --profiles to list profile names for the game instead of mods. From a27cc482aa61f557c97dcd4cdd80261fa12b508f Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sun, 2 Aug 2026 00:27:47 -0400 Subject: [PATCH 88/96] fix: correct field-vs-row and load-order wording (#201 review) - cmd/lmm/list.go help text said a later mod "wins conflicting rows" - the actual merge semantics are field-level (a per-field upsert on a shared row, not a whole-row overwrite), matching the README/ configuration.md wording already written for #201. Ran `make man` to regenerate the now-stale lmm-list.1 page. - list_order_test.go's assertion messages labeled Mod B as "later in load order" when the test's own ReorderMods call ([B, A]) actually makes Mod A last (final say) and Mod B first (lowest priority) - a failure would have pointed at the wrong mod. Reworded to state the actual array order and which mod has final say. --- cmd/lmm/list.go | 8 +++++--- cmd/lmm/list_order_test.go | 4 ++-- docs/man/man1/lmm-list.1 | 8 +++++--- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/cmd/lmm/list.go b/cmd/lmm/list.go index faa28e8..680d035 100644 --- a/cmd/lmm/list.go +++ b/cmd/lmm/list.go @@ -50,9 +50,11 @@ var listCmd = &cobra.Command{ Mods are printed in the profile's load order (see 'lmm profile reorder') - the same order that decides merge precedence for a compiled/merged pak: -a mod later in the load order is merged later and wins conflicting rows. -A mod installed but missing from the load order is still shown (never -silently dropped), placed first since it has no claim to the final say. +a mod later in the load order is merged later and wins conflicting +fields on a shared data-table row (untouched fields from earlier mods +still survive). A mod installed but missing from the load order is +still shown (never silently dropped), placed first since it has no +claim to the final say. Use --profiles to list profile names for the game instead of mods. diff --git a/cmd/lmm/list_order_test.go b/cmd/lmm/list_order_test.go index a338ee8..1339443 100644 --- a/cmd/lmm/list_order_test.go +++ b/cmd/lmm/list_order_test.go @@ -49,13 +49,13 @@ func TestList_DisplaysProfileLoadOrder_NotInstallOrder(t *testing.T) { t.Run("non-verbose", func(t *testing.T) { out := listNonVerbose(t, svc, game) idx := modOrder(t, out, "Mod B", "Mod A") - assert.Less(t, idx[0], idx[1], "Mod B (later in load order) must print before Mod A") + assert.Less(t, idx[0], idx[1], "profile load order is [Mod B, Mod A] (Mod A is last - final say); the listing must follow that array order, not install order (which was A then B)") }) t.Run("verbose", func(t *testing.T) { out := listVerbose(t, svc, game, false) idx := modOrder(t, out, "Mod B", "Mod A") - assert.Less(t, idx[0], idx[1], "Mod B (later in load order) must print before Mod A") + assert.Less(t, idx[0], idx[1], "profile load order is [Mod B, Mod A] (Mod A is last - final say); the listing must follow that array order, not install order (which was A then B)") }) t.Run("json", func(t *testing.T) { diff --git a/docs/man/man1/lmm-list.1 b/docs/man/man1/lmm-list.1 index 735d1bb..f6670fe 100644 --- a/docs/man/man1/lmm-list.1 +++ b/docs/man/man1/lmm-list.1 @@ -15,9 +15,11 @@ List all mods installed in the specified game and profile. .PP Mods are printed in the profile's load order (see 'lmm profile reorder') - the same order that decides merge precedence for a compiled/merged pak: -a mod later in the load order is merged later and wins conflicting rows. -A mod installed but missing from the load order is still shown (never -silently dropped), placed first since it has no claim to the final say. +a mod later in the load order is merged later and wins conflicting +fields on a shared data-table row (untouched fields from earlier mods +still survive). A mod installed but missing from the load order is +still shown (never silently dropped), placed first since it has no +claim to the final say. .PP Use --profiles to list profile names for the game instead of mods. From 4e410834d7b58fe558d4cd9b12eaf63276a41bc4 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sun, 2 Aug 2026 00:29:34 -0400 Subject: [PATCH 89/96] docs: archive completed Icarus epic plans (#136, #175, #197) --- ...2026-07-29-icarus-exmod-pak-compilation.md | 4449 +++++++++++++++++ .../2026-07-29-icarus-exmod-pak-research.md | 91 + .../archive/2026-08-01-icarus-merged-pak.md | 3299 ++++++++++++ ...6-08-01-icarus-quickbms-fallback-design.md | 62 + .../2026-08-01-icarus-quickbms-fallback.md | 2024 ++++++++ .../archive/2026-08-01-icarus-zlib-pivot.md | 1409 ++++++ .../archive/icarus-pak-format-findings.md | 852 ++++ .../archive/icarus-quickbms-spike-findings.md | 327 ++ 8 files changed, 12513 insertions(+) create mode 100644 docs/plans/archive/2026-07-29-icarus-exmod-pak-compilation.md create mode 100644 docs/plans/archive/2026-07-29-icarus-exmod-pak-research.md create mode 100644 docs/plans/archive/2026-08-01-icarus-merged-pak.md create mode 100644 docs/plans/archive/2026-08-01-icarus-quickbms-fallback-design.md create mode 100644 docs/plans/archive/2026-08-01-icarus-quickbms-fallback.md create mode 100644 docs/plans/archive/2026-08-01-icarus-zlib-pivot.md create mode 100644 docs/plans/archive/icarus-pak-format-findings.md create mode 100644 docs/plans/archive/icarus-quickbms-spike-findings.md diff --git a/docs/plans/archive/2026-07-29-icarus-exmod-pak-compilation.md b/docs/plans/archive/2026-07-29-icarus-exmod-pak-compilation.md new file mode 100644 index 0000000..a34e142 --- /dev/null +++ b/docs/plans/archive/2026-07-29-icarus-exmod-pak-compilation.md @@ -0,0 +1,4449 @@ +# Icarus `.exmod`/`.exmodz` PAK Compilation Implementation Plan + +> **SUPERSEDED IN PART (2026-08-01, #175).** Everywhere this plan states that `data.pak`'s +> tables are Oodle-compressed and therefore unreadable — the Global Constraints' network +> bullet, Task 12's base-table note, Task 12a (the dump fetcher) and Task 13's `data_dump_path` +> wiring — the premise is false: those tables are **Zlib**, which the standard library reads. +> Tasks 1–11 (the pak format work, the Firestore source, `.EXMOD`/`.EXMODZ` handling) are +> unaffected and shipped as written. The dump subsystem those later tasks built has been +> removed by [`2026-08-01-icarus-zlib-pivot.md`](2026-08-01-icarus-zlib-pivot.md); read that +> for the current design. This document is kept unedited below as the record of how the epic +> was actually built. + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let LMM install Icarus mods distributed as `.exmodz` by compiling their JSON diff into a working `_P.pak` on Linux, end to end from browsing the Firestore-backed catalog through deployed pak. + +**Architecture:** A game-agnostic `internal/unrealpak` package (UE 4.25–4.27-range PAK index reader/writer, uncompressed+unencrypted only) underpins an Icarus-specific `internal/source/icarus` package (Firestore REST `ModSource` + `.EXMOD` diff engine + `.EXMODZ` unpacking + compile orchestration). A new `domain.DeployCompile` mode and a `source.Compiler` optional capability wire the compile step into `Service`'s existing cache-population path, so the linker's deploy step needs no changes at all. + +**Tech Stack:** Go 1.25 (this repo's existing version), stdlib only (`net/http`, `encoding/json`, `encoding/binary`, `archive/zip`, `crypto/sha1`) — no new third-party dependencies, matching this repo's existing NexusMods/CurseForge clients and its `modernc.org/sqlite`-style no-CGO/no-external-binary convention. + +## Global Constraints + +- No new third-party Go dependencies (`go.mod` stays free of a Firestore SDK, a PAK library, etc.) — plain `net/http` and stdlib only, per repo convention and `~/.claude/GO.md`. +- No silent fallbacks: unexpected PAK format, encryption, or compression fails loudly with a clear error (repo precedent: #95). +- v1 PAK reader/writer scope is **uncompressed, unencrypted only** — anything else is a hard error, not a degraded path. +- **The compile path requires network access** (user decision, rev3): base data tables come + from the community's hosted per-week JSON dumps, not from decompressing the local + `data.pak`. This is a deliberate exception to compiling from local data only, forced by + Oodle — 258 of `data.pak`'s 298 tables are Oodle-compressed and no stdlib decoder exists. + Stdlib-only still holds (`net/http` is fine; no new dependencies). Compiling offline is + therefore not supported; that must surface as a clear error, never a silent skip. +- Design source of truth: [`docs/plans/2026-07-29-icarus-exmod-pak-research.md`](../../plans/2026-07-29-icarus-exmod-pak-research.md) on branch `docs/icarus-exmod-pak-research`. This plan builds on that branch. +- Tracked by [#136](https://github.com/DonovanMods/linux-mod-manager/issues/136) — reference it in commit messages/PR per repo workflow. +- Follow this repo's TDD/table-driven Go conventions (`~/.claude/GO.md`, `~/.claude/DEV.md`) throughout. + +--- + +## Task 1: Validate the real PAK footer/index format against a local install + +This is an empirical spike, not code — everything downstream depends on its findings. The plan assumes the documented classic UE4 `FPakInfo` footer layout (magic `0x5A6F12E1`, version-gated fields), which is well established publicly (`repak`, `u4pak`) but has **not** been confirmed against Icarus's actual bytes. + +**Files:** + +- Create: `docs/plans/icarus-pak-format-findings.md` (scratch findings doc, committed alongside Task 2 once confirmed — not shipped as product docs) + +**Interfaces:** + +- Produces: confirmed values for `footerSize` (221 or 61 bytes — see Task 2), `Version` (int32), `bEncryptedIndex` (expected `false`), that later tasks' hard-coded assumptions in `internal/unrealpak` must match. + +> **STATUS: DONE — and it falsified this plan's index assumption.** Both spike rounds are +> complete; findings are in [`docs/plans/icarus-pak-format-findings.md`](icarus-pak-format-findings.md). +> Steps 1–6 below are kept for provenance but need not be re-run. Three results reshaped +> Tasks 2–5, 12 and 13: +> +> 1. **The footer is confirmed** (version 11, 221 bytes, `bEncryptedIndex == 0`, SHA1 match) +> — verified on all 34 paks in the install. +> 2. **The index is NOT the classic flat `MountPoint`+`NumEntries`+N×`FPakEntry` layout this +> plan assumed.** Version 11 uses the UE 4.25+ three-part index: a primary index carrying +> _bit-packed_ entry records plus SHA1-gated offsets to a path-hash index and a full +> directory index. Every structure is now decoded at byte level in the findings doc +> (Part 2), verified against 173,078 real entries. +> 3. **The `.EXMOD` base pak is `Icarus/Content/Data/data.pak`, not a pakchunk** — 298 files, +> all `.json`. The `Content/Paks/pakchunk0*` chunks contain zero `.json`. See the Oodle +> blocker note in Task 12. + +- [ ] **Step 1: Locate the local `data.pak`** + +```bash +find ~/.steam ~/.local/share/Steam -ipath "*Icarus*Content/Paks/*.pak" 2>/dev/null +``` + +Expected: a path like `.../steamapps/common/Icarus/Icarus/Content/Paks/pakchunk0-WindowsNoEditor.pak` (or similarly named — Icarus ships its base data under one or more numbered pakchunks, not necessarily literally named `data.pak`; note the exact filename(s) found). + +- [ ] **Step 2: Dump the last 256 bytes and locate the magic** + +```bash +PAK=/path/found/above +tail -c 256 "$PAK" | xxd | tail -20 +python3 -c " +import struct +data = open('$PAK','rb').read()[-256:] +magic = struct.pack(' footer size:', len(data)-off) +# bEncryptedIndex is the byte immediately after IndexHash, present for version>=4 +enc_off = off+4+4+8+8+20 +print('bEncryptedIndex byte:', data[enc_off]) +" +``` + +Expected (per Task 2's assumptions): `bEncryptedIndex == 0`. Record the actual `version` and footer size (221 vs 61 vs other) in `docs/plans/icarus-pak-format-findings.md`. + +**Note (added post-spike):** the `bEncryptedIndex` byte offset in this script's naive parse (`off+4+4+8+8+20`, i.e. immediately after `IndexHash`) is **wrong** for Icarus's real version-11 paks — it lands inside the trailing `CompressionMethods` table instead of the actual flag byte. The real layout has `EncryptionKeyGuid`(16 bytes) + `bEncryptedIndex`(1 byte) immediately **before** `Magic`, and (for version≥8) a `CompressionMethods` table **after** `IndexHash`. The corrected byte layout is documented in `docs/plans/icarus-pak-format-findings.md`, which supersedes this script's field-offset assumptions — Task 2's implementation uses the corrected layout, not this script's. + +- [ ] **Step 4: Cross-check IndexHash against the actual index bytes** + +```bash +python3 -c " +import hashlib +data = open('$PAK', 'rb').read() +index_offset = +index_size = +index_bytes = data[index_offset:index_offset+index_size] +print('sha1 matches footer IndexHash:', hashlib.sha1(index_bytes).hexdigest()) +" +``` + +Compare against the `index_hash` hex from Step 3. A match is strong confirmation the offset/size/version parsing above is correct — this is the acceptance gate for Task 2's format assumptions, since a wrong version/field-width would make this hash disagree. + +- [ ] **Step 5: Write up findings** + +Create `docs/plans/icarus-pak-format-findings.md` with: exact pak filename(s) found, `version`, footer size, confirmed `bEncryptedIndex == false`, and the SHA1 cross-check result from Step 4. If any assumption in Task 2 turns out wrong (different footer size, encrypted index, unexpected version), stop and revise Task 2's `footerSizes`/version-gate constants before proceeding — do not implement Task 2 against unconfirmed values. + +- [ ] **Step 6: Commit** + +```bash +git add docs/plans/icarus-pak-format-findings.md +git commit -m "docs: confirm Icarus data.pak footer format (#136)" +``` + +--- + +## Task 2: `internal/unrealpak` — footer + index reader + +**Files:** + +- Create: `internal/unrealpak/pak.go` (shared types/errors) +- Create: `internal/unrealpak/reader.go` +- Create: `internal/unrealpak/reader_test.go` + +**Interfaces:** + +- Consumes: footer format confirmed in Task 1. +- Produces: `type Reader struct{...}`, `func Open(path string) (*Reader, error)`, `func (r *Reader) Close() error`, `func (r *Reader) Files() []FileEntry`, `type FileEntry struct { Path string; Size int64 }` — Task 3 and Task 12 depend on these exact names. + +- [ ] **Step 1: Write `pak.go` shared types, constants and format primitives** + +```go +package unrealpak + +import ( + "bytes" + "crypto/sha1" //nolint:gosec // pak format uses SHA1, not our choice + "encoding/binary" + "errors" + "strings" + "unicode/utf16" +) + +// ErrUnsupportedFormat indicates the pak uses a feature this package +// deliberately does not support (compression, encryption, exotic FString +// encodings) rather than a genuine parse failure. Callers should fail loudly +// on this, not silently degrade (repo precedent: #95). +var ErrUnsupportedFormat = errors.New("unrealpak: unsupported pak feature") + +const magic uint32 = 0x5A6F12E1 + +// footerSize is the only footer shape this package supports: the version>=8 +// layout, EncryptionKeyGuid(16)+bEncryptedIndex(1)+Magic(4)+Version(4)+ +// IndexOffset(8)+IndexSize(8)+IndexHash(20)+CompressionMethods(5x32) = 221. +// Note EncryptionKeyGuid and bEncryptedIndex precede Magic — not after +// IndexHash, as some public docs describe. Confirmed on all 34 paks in a real +// Icarus install (Task 1); see docs/plans/icarus-pak-format-findings.md. +const footerSize = 221 + +// minVersion is the oldest pak version this package reads. Version 10 +// (PakFile_Version_PathHashIndex) introduced the three-part index — primary +// index + path-hash index + full directory index — that this package parses. +// Older paks use a flat index with a completely different shape; rather than +// carry a second parser for a layout Icarus does not ship, they are a hard +// ErrUnsupportedFormat (repo precedent #95: no silent fallbacks). +const minVersion int32 = 10 + +// writeVersion is what Writer emits: the same version Icarus's own paks use, +// so the engine loads our output through the exact code path it already uses. +const writeVersion int32 = 11 + +// storedHeaderSize is the on-disk size of the per-entry FPakEntry header that +// precedes each stored (uncompressed) file's payload: +// Offset(8)+Size(8)+UncompressedSize(8)+CompressionMethodIndex(4)+Hash(20)+ +// Flags(1)+CompressionBlockSize(4) = 53. Compressed entries add +// BlockCount(4)+16*blocks between Hash and Flags; this package never writes +// those and refuses to read their payloads. +const storedHeaderSize = 53 + +// FileEntry describes one file inside a pak, as returned by Reader.Files. +type FileEntry struct { + Path string // Mount-relative path, e.g. "Icarus/Content/Data/AI-D_AIGrowth.json" + Size int64 // Uncompressed size in bytes +} + +// hashPath computes a path's key in the pak's path-hash index: FNV-1a 64 over +// the UTF-16LE bytes of the lowercased mount-relative path (no NUL +// terminator), seeded by ADDING the pak's PathHashSeed to the FNV offset +// basis. Any leading "/" is stripped first — the full directory index stores +// root-level files under a "/" directory, and the hash is taken over the path +// without it. +// +// This recipe was not guessed: it was recovered by brute-forcing seed/ +// encoding/case/prefix combinations until computed hashes matched stored keys, +// then verified against all 173,078 entries across all 34 paks in a real +// install. See docs/plans/icarus-pak-format-findings.md. +// +// strings.ToLower is full-Unicode where UE's FChar::ToLower is not, but no +// non-ASCII path exists in any shipped Icarus pak and this package controls +// the paths it writes, so the two agree for everything we handle. +func hashPath(mountRelative string, seed uint64) uint64 { + const ( + offsetBasis uint64 = 0xCBF29CE484222325 + prime uint64 = 0x00000100000001B3 + ) + h := offsetBasis + seed + for _, u := range utf16.Encode([]rune(strings.ToLower(strings.TrimPrefix(mountRelative, "/")))) { + h ^= uint64(byte(u)) + h *= prime + h ^= uint64(byte(u >> 8)) + h *= prime + } + return h +} +``` + +Then the shared **format primitives** — the byte-emitters for the four structures a +version-11 pak is made of. They live here rather than in `writer.go` because Task 2's +reader test builds its own fixture pak with them and must not depend on Task 4: + +```go +// defaultMountPoint is the mount point Writer stamps into the primary index. +// Icarus's own data.pak uses an absolute cook-machine path +// ("C:/BA/work/.../Temp/Data/"); "../../../" is the conventional relative form +// used by its pakchunks. Confirming which one a _P.pak needs to override +// Content/Data/data.pak in-game is a post-plan validation item. +const defaultMountPoint = "../../../" + +// writeFString writes a length-prefixed ANSI Unreal FString (length includes +// the trailing NUL). +func writeFString(buf *bytes.Buffer, s string) { + b := append([]byte(s), 0) + binary.Write(buf, binary.LittleEndian, int32(len(b))) //nolint:errcheck // bytes.Buffer writes never fail + buf.Write(b) +} + +// splitMountPath splits a mount-relative path into the directory-index key +// (trailing "/", or exactly "/" for a root-level file) and the leaf name, +// matching how real paks key their directory indexes. +func splitMountPath(rel string) (dir, file string) { + if i := strings.LastIndex(rel, "/"); i >= 0 { + return rel[:i+1], rel[i+1:] + } + return "/", rel +} + +// storedEntryHeader builds the 53-byte FPakEntry header that precedes a stored +// file's payload on disk. The Offset field is always 0 in this local copy — +// real paks write 0 there too, the authoritative offset lives in the index. +// Hash is the SHA1 of the on-disk payload bytes. +func storedEntryHeader(size int64, content []byte) []byte { + var b bytes.Buffer + binary.Write(&b, binary.LittleEndian, int64(0)) //nolint:errcheck // Offset + binary.Write(&b, binary.LittleEndian, size) //nolint:errcheck // Size + binary.Write(&b, binary.LittleEndian, size) //nolint:errcheck // UncompressedSize + binary.Write(&b, binary.LittleEndian, int32(0)) //nolint:errcheck // CompressionMethodIndex: stored + h := sha1.Sum(content) //nolint:gosec + b.Write(h[:]) + b.WriteByte(0) // Flags: not encrypted, not deleted + binary.Write(&b, binary.LittleEndian, uint32(0)) //nolint:errcheck // CompressionBlockSize + return b.Bytes() +} + +// buildPrimaryIndex serializes the primary index. Callers build it twice: the +// sub-index offsets it records point past its own end, but its length does not +// depend on their values (they are fixed-width int64), so a first pass with +// zero offsets measures it and a second pass writes the real ones. +func buildPrimaryIndex(numEntries int32, seed uint64, + phiOffset, phiSize int64, phiHash [20]byte, + fdiOffset, fdiSize int64, fdiHash [20]byte, encoded []byte) []byte { + var b bytes.Buffer + writeFString(&b, defaultMountPoint) + binary.Write(&b, binary.LittleEndian, numEntries) //nolint:errcheck + binary.Write(&b, binary.LittleEndian, seed) //nolint:errcheck // PathHashSeed + binary.Write(&b, binary.LittleEndian, int32(1)) //nolint:errcheck // bHasPathHashIndex + binary.Write(&b, binary.LittleEndian, phiOffset) //nolint:errcheck + binary.Write(&b, binary.LittleEndian, phiSize) //nolint:errcheck + b.Write(phiHash[:]) + binary.Write(&b, binary.LittleEndian, int32(1)) //nolint:errcheck // bHasFullDirectoryIndex + binary.Write(&b, binary.LittleEndian, fdiOffset) //nolint:errcheck + binary.Write(&b, binary.LittleEndian, fdiSize) //nolint:errcheck + b.Write(fdiHash[:]) + binary.Write(&b, binary.LittleEndian, int32(len(encoded))) //nolint:errcheck // EncodedPakEntriesSize + b.Write(encoded) + binary.Write(&b, binary.LittleEndian, int32(0)) //nolint:errcheck // NumNonEncodedFiles: none + return b.Bytes() +} + +// buildFooter serializes the 221-byte version>=8 footer. +func buildFooter(version int32, indexOffset, indexSize int64, indexHash [20]byte) []byte { + var b bytes.Buffer + b.Write(make([]byte, 16)) // EncryptionKeyGuid: zero + b.WriteByte(0) // bEncryptedIndex: false + binary.Write(&b, binary.LittleEndian, magic) //nolint:errcheck + binary.Write(&b, binary.LittleEndian, version) //nolint:errcheck + binary.Write(&b, binary.LittleEndian, indexOffset) //nolint:errcheck + binary.Write(&b, binary.LittleEndian, indexSize) //nolint:errcheck + b.Write(indexHash[:]) + // CompressionMethods: 5 fixed-width 32-byte name slots, all empty since + // this package only ever writes stored entries. Real paks name "Oodle" + // and "Zlib" here; an all-zero table is the correct shape for method 0. + b.Write(make([]byte, 160)) + return b.Bytes() +} +``` + +(Both blocks above are one file: `pak.go`'s import list at the top covers them.) + +- [ ] **Step 2: Write the failing test for footer parsing** + +```go +package unrealpak + +import ( + "bytes" + "crypto/sha1" //nolint:gosec // pak format uses SHA1, not our choice + "encoding/binary" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +// fixtureSeed is an arbitrary PathHashSeed for fixture paks. Readers take the +// seed from the index, so any value works — real paks use a different one per +// chunk. +const fixtureSeed uint64 = 0x0123456789ABCDEF + +// writeMinimalPak builds a hand-crafted but fully valid version-11 pak holding +// a single stored entry: data section, primary index, path-hash index, full +// directory index, then the 221-byte footer. It deliberately does not use the +// Task 4 Writer — the reader's tests must be able to fail independently of the +// writer, and vice versa. +func writeMinimalPak(t *testing.T, mountPath string, content []byte) string { + t.Helper() + return writeMinimalPakMethod(t, mountPath, content, 0) +} + +// writeMinimalPakMethod builds a fixture whose entry claims CompressionMethodIndex +// method. Only method 0 produces a genuinely readable pak; non-zero values exist +// to exercise the reader's refusal path (Task 3), which is the case that matters +// in practice — 74% of real Icarus entries are Oodle-compressed. +func writeMinimalPakMethod(t *testing.T, mountPath string, content []byte, method int32) string { + t.Helper() + pakPath := filepath.Join(t.TempDir(), "test.pak") + if err := os.WriteFile(pakPath, buildFixturePak(mountPath, content, method), 0o644); err != nil { + t.Fatalf("writing test pak: %v", err) + } + return pakPath +} + +func buildFixturePak(mountPath string, content []byte, method int32) []byte { + rel := strings.TrimPrefix(mountPath, "/") + + // Data section: the 53-byte per-entry header, then the payload, at offset 0. + var data bytes.Buffer + hdr := storedEntryHeader(int64(len(content)), content) + binary.LittleEndian.PutUint32(hdr[24:28], uint32(method)) // CompressionMethodIndex + data.Write(hdr) + data.Write(content) + + // One encoded index record. For method 0 that is the 12-byte stored shape: + // flags 0xE0000000 (offset/uncompressed-size/size all 32-bit-safe, no + // blocks), then uint32 Offset and uint32 UncompressedSize — Size is not + // serialized for method 0, it equals UncompressedSize. A non-zero method + // adds the uint32 Size field, per the encoded-record layout. + var encoded bytes.Buffer + binary.Write(&encoded, binary.LittleEndian, uint32(0xE0000000)|uint32(method)<<23) //nolint:errcheck + binary.Write(&encoded, binary.LittleEndian, uint32(0)) //nolint:errcheck + binary.Write(&encoded, binary.LittleEndian, uint32(len(content))) //nolint:errcheck + if method != 0 { + binary.Write(&encoded, binary.LittleEndian, uint32(len(content))) //nolint:errcheck // Size + } + + // Full directory index: one directory, one file, pointing at blob offset 0. + dirName, fileName := splitMountPath(rel) + var fdi bytes.Buffer + binary.Write(&fdi, binary.LittleEndian, int32(1)) //nolint:errcheck // DirCount + writeFString(&fdi, dirName) + binary.Write(&fdi, binary.LittleEndian, int32(1)) //nolint:errcheck // FileCount + writeFString(&fdi, fileName) + binary.Write(&fdi, binary.LittleEndian, int32(0)) //nolint:errcheck // PakEntryLocation + + // Path-hash index: the hash->location map, then an EMPTY pruned directory + // index. 33 of the 34 paks in a real install ship it empty, so a bare + // int32(0) is a shape the engine demonstrably accepts. + var phi bytes.Buffer + binary.Write(&phi, binary.LittleEndian, int32(1)) //nolint:errcheck // Count + binary.Write(&phi, binary.LittleEndian, hashPath(rel, fixtureSeed)) //nolint:errcheck + binary.Write(&phi, binary.LittleEndian, int32(0)) //nolint:errcheck // location + binary.Write(&phi, binary.LittleEndian, int32(0)) //nolint:errcheck // pruned index: 0 dirs + + phiHash := sha1.Sum(phi.Bytes()) //nolint:gosec + fdiHash := sha1.Sum(fdi.Bytes()) //nolint:gosec + + indexOffset := int64(data.Len()) + sizing := buildPrimaryIndex(1, fixtureSeed, 0, 0, phiHash, 0, 0, fdiHash, encoded.Bytes()) + phiOffset := indexOffset + int64(len(sizing)) + fdiOffset := phiOffset + int64(phi.Len()) + index := buildPrimaryIndex(1, fixtureSeed, + phiOffset, int64(phi.Len()), phiHash, + fdiOffset, int64(fdi.Len()), fdiHash, encoded.Bytes()) + indexHash := sha1.Sum(index) //nolint:gosec + + var out bytes.Buffer + out.Write(data.Bytes()) + out.Write(index) + out.Write(phi.Bytes()) + out.Write(fdi.Bytes()) + out.Write(buildFooter(writeVersion, indexOffset, int64(len(index)), indexHash)) + return out.Bytes() +} + +func TestReader_Open_ListsFiles(t *testing.T) { + content := []byte(`{"hello":"world"}`) + path := writeMinimalPak(t, "Icarus/Content/Data/Test.json", content) + + r, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() + + files := r.Files() + if len(files) != 1 { + t.Fatalf("got %d files, want 1", len(files)) + } + if files[0].Path != "Icarus/Content/Data/Test.json" { + t.Errorf("Path = %q, want Icarus/Content/Data/Test.json", files[0].Path) + } + if files[0].Size != int64(len(content)) { + t.Errorf("Size = %d, want %d", files[0].Size, len(content)) + } +} + +// A root-level file is keyed under the "/" directory in the directory index; +// Files must report it without the leading slash, matching what hashPath uses. +func TestReader_Open_RootLevelFile(t *testing.T) { + path := writeMinimalPak(t, "x.json", []byte("{}")) + r, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() + + files := r.Files() + if len(files) != 1 || files[0].Path != "x.json" { + t.Fatalf("Files() = %+v, want one entry with Path %q", files, "x.json") + } +} + +func TestReader_Open_RejectsEncryptedIndex(t *testing.T) { + path := writeMinimalPak(t, "x.json", []byte("{}")) + data, _ := os.ReadFile(path) + // bEncryptedIndex sits at offset 16 from footer start — right after the + // 16-byte EncryptionKeyGuid, immediately before Magic. + data[len(data)-footerSize+16] = 1 + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatal(err) + } + + _, err := Open(path) + if !errors.Is(err, ErrUnsupportedFormat) { + t.Fatalf("Open error = %v, want ErrUnsupportedFormat", err) + } +} + +// Versions below 10 use a flat index this package deliberately does not parse: +// a hard error, never a fallback. +func TestReader_Open_RejectsPreVersion10(t *testing.T) { + path := writeMinimalPak(t, "x.json", []byte("{}")) + data, _ := os.ReadFile(path) + // Version is the int32 at footer offset 21 (after Guid+flag+Magic). + binary.LittleEndian.PutUint32(data[len(data)-footerSize+21:], uint32(9)) + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatal(err) + } + + _, err := Open(path) + if !errors.Is(err, ErrUnsupportedFormat) { + t.Fatalf("Open error = %v, want ErrUnsupportedFormat", err) + } +} + +// Corruption anywhere in the index must trip a SHA1 gate rather than be +// parsed. The full directory index is the last region before the footer, so +// flipping the byte just before it exercises the primary->sub-index gate. +func TestReader_Open_RejectsCorruptedDirectoryIndex(t *testing.T) { + path := writeMinimalPak(t, "x.json", []byte("{}")) + data, _ := os.ReadFile(path) + data[len(data)-footerSize-1] ^= 0xFF + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatal(err) + } + + if _, err := Open(path); err == nil { + t.Fatal("expected error for corrupted directory index, got nil") + } +} +``` + +- [ ] **Step 3: Run to verify it fails** + +```bash +go test ./internal/unrealpak/... -run TestReader_Open -v +``` + +Expected: FAIL (`Open` undefined). + +- [ ] **Step 4: Implement `reader.go`** + +```go +package unrealpak + +import ( + "bytes" + "crypto/sha1" //nolint:gosec // pak format uses SHA1, not our choice + "encoding/binary" + "fmt" + "io" + "os" + "sort" + "strings" +) + +// Reader provides read access to an uncompressed, unencrypted UE4-range pak. +type Reader struct { + f *os.File + entries []readerEntry +} + +type readerEntry struct { + FileEntry + offset int64 // absolute offset of the entry's on-disk header + method int32 // CompressionMethodIndex; 0 = stored. Non-zero entries are + // enumerated but their payloads cannot be read (see ReadFile, Task 3). +} + +// Open parses path's footer and index. It does not read file contents — +// call ReadFile for that (Task 3). +func Open(path string) (*Reader, error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("unrealpak: opening %s: %w", path, err) + } + info, err := f.Stat() + if err != nil { + f.Close() //nolint:errcheck + return nil, fmt.Errorf("unrealpak: stat %s: %w", path, err) + } + + ft, err := readFooter(f, info.Size()) + if err != nil { + f.Close() //nolint:errcheck + return nil, err + } + if ft.encryptedIndex { + f.Close() //nolint:errcheck + return nil, fmt.Errorf("unrealpak: %s: %w: encrypted index", path, ErrUnsupportedFormat) + } + + indexBuf, err := readRegion(f, ft.indexOffset, ft.indexSize, ft.indexHash) + if err != nil { + f.Close() //nolint:errcheck + return nil, fmt.Errorf("unrealpak: %s: primary index: %w", path, err) + } + + entries, err := parseIndex(f, indexBuf) + if err != nil { + f.Close() //nolint:errcheck + return nil, fmt.Errorf("unrealpak: %s: parsing index: %w", path, err) + } + + return &Reader{f: f, entries: entries}, nil +} + +// readRegion reads size bytes at offset and verifies them against want. Every +// index region in a version-11 pak is SHA1-gated: the footer covers the +// primary index, and the primary index covers each sub-index. All three gates +// are enforced — a mismatch is corruption or an unrecognized layout, never +// something to parse through. +func readRegion(r io.ReaderAt, offset, size int64, want [20]byte) ([]byte, error) { + if offset < 0 || size < 0 { + return nil, fmt.Errorf("%w: negative region offset/size", ErrUnsupportedFormat) + } + buf := make([]byte, size) + if _, err := r.ReadAt(buf, offset); err != nil { + return nil, fmt.Errorf("reading region at %d: %w", offset, err) + } + if sum := sha1.Sum(buf); !bytes.Equal(sum[:], want[:]) { //nolint:gosec + return nil, fmt.Errorf("hash mismatch (corrupt or unsupported format)") + } + return buf, nil +} + +// Close releases the underlying file handle. +func (r *Reader) Close() error { return r.f.Close() } + +// Files returns every file this pak's index describes. +func (r *Reader) Files() []FileEntry { + out := make([]FileEntry, len(r.entries)) + for i, e := range r.entries { + out[i] = e.FileEntry + } + return out +} + +type footer struct { + version int32 + indexOffset int64 + indexSize int64 + indexHash [20]byte + encryptedIndex bool +} + +// readFooter parses the single 221-byte footer shape this package supports. +// The footer is fixed-size and sits flush against EOF, so there is nothing to +// search for and no alternate width to try: if Magic isn't where it must be, +// this is not a pak we handle. +func readFooter(r io.ReaderAt, fileSize int64) (footer, error) { + if fileSize < footerSize { + return footer{}, fmt.Errorf("%w: file of %d bytes is smaller than a %d-byte footer", + ErrUnsupportedFormat, fileSize, footerSize) + } + buf := make([]byte, footerSize) + if _, err := r.ReadAt(buf, fileSize-footerSize); err != nil { + return footer{}, fmt.Errorf("reading footer: %w", err) + } + // Layout: EncryptionKeyGuid(0:16) bEncryptedIndex(16) Magic(17:21) + // Version(21:25) IndexOffset(25:33) IndexSize(33:41) IndexHash(41:61) + // CompressionMethods(61:221). + if binary.LittleEndian.Uint32(buf[17:21]) != magic { + return footer{}, fmt.Errorf("%w: no pak magic at the expected footer offset", ErrUnsupportedFormat) + } + ft := footer{ + encryptedIndex: buf[16] != 0, + version: int32(binary.LittleEndian.Uint32(buf[21:25])), + indexOffset: int64(binary.LittleEndian.Uint64(buf[25:33])), + indexSize: int64(binary.LittleEndian.Uint64(buf[33:41])), + } + copy(ft.indexHash[:], buf[41:61]) + if ft.version < minVersion { + return footer{}, fmt.Errorf("%w: pak version %d (this package requires >= %d)", + ErrUnsupportedFormat, ft.version, minVersion) + } + // The trailing CompressionMethods name table is intentionally left + // unparsed: entries carry a method *index*, and this package only ever + // reads payloads whose index is 0 (stored), which needs no name. + return ft, nil +} + +// parseIndex parses the primary index, then the full directory index it points +// at, resolving every path to its bit-packed entry record. +// +// Version-11 paks have no flat entry array. The primary index holds a blob of +// bit-packed records plus SHA1-gated offsets to two sub-indexes: a path-hash +// index (hash -> record offset) and a full directory index +// (directory -> file -> record offset). Enumeration uses the directory index, +// which is the only one that carries real path strings. +func parseIndex(f io.ReaderAt, index []byte) ([]readerEntry, error) { + c := &cursor{b: index} + c.fstring() // MountPoint: recorded for the engine's benefit, unused here + numEntries := c.i32() + seed := c.u64() + _ = seed // only the writer needs the seed; enumeration goes via the directory index + + pathHash, err := readSubIndexRef(c, "path hash index") + if err != nil { + return nil, err + } + fullDir, err := readSubIndexRef(c, "full directory index") + if err != nil { + return nil, err + } + encoded := c.bytes(int(c.i32())) // EncodedPakEntriesSize, then the blob + if nonEncoded := c.i32(); nonEncoded != 0 { + return nil, fmt.Errorf("%w: %d non-encoded index entries", ErrUnsupportedFormat, nonEncoded) + } + if c.err != nil { + return nil, fmt.Errorf("primary index: %w", c.err) + } + + // Verify the path-hash index's hash even though enumeration does not use + // it: it is part of the format's integrity chain, and a pak whose + // sub-index hashes don't hold is not one to trust. + if _, err := readRegion(f, pathHash.offset, pathHash.size, pathHash.hash); err != nil { + return nil, fmt.Errorf("path hash index: %w", err) + } + dirBuf, err := readRegion(f, fullDir.offset, fullDir.size, fullDir.hash) + if err != nil { + return nil, fmt.Errorf("full directory index: %w", err) + } + + entries, err := parseDirectoryIndex(dirBuf, encoded) + if err != nil { + return nil, err + } + if int32(len(entries)) != numEntries { + return nil, fmt.Errorf("directory index lists %d files, index header says %d", + len(entries), numEntries) + } + sort.Slice(entries, func(i, j int) bool { return entries[i].Path < entries[j].Path }) + return entries, nil +} + +type subIndexRef struct { + offset, size int64 + hash [20]byte +} + +// readSubIndexRef reads a `bHasIndex` flag and, when set, the offset/size/ +// hash triple that follows. Both sub-indexes are required: every pak version +// this package accepts writes both, and a reader that limped along without the +// directory index would have no paths to report. +func readSubIndexRef(c *cursor, name string) (subIndexRef, error) { + if c.i32() == 0 { + return subIndexRef{}, fmt.Errorf("%w: pak has no %s", ErrUnsupportedFormat, name) + } + ref := subIndexRef{offset: c.i64(), size: c.i64()} + copy(ref.hash[:], c.bytes(20)) + return ref, c.err +} + +// parseDirectoryIndex walks directory -> file -> entry-location and decodes the +// bit-packed record each location points at. +func parseDirectoryIndex(dir, encoded []byte) ([]readerEntry, error) { + c := &cursor{b: dir} + dirCount := c.i32() + var entries []readerEntry + for i := int32(0); i < dirCount && c.err == nil; i++ { + dirName := c.fstring() + fileCount := c.i32() + for j := int32(0); j < fileCount && c.err == nil; j++ { + fileName := c.fstring() + loc := c.i32() + // Root-level files live under a "/" directory key, so the naive + // join yields a leading slash; the canonical mount-relative path + // (and the one hashPath consumes) has none. + full := strings.TrimPrefix(dirName+fileName, "/") + if loc < 0 { + // Negative locations index a non-encoded FPakEntry array. No + // pak in a real Icarus install uses them. + return nil, fmt.Errorf("entry %q: %w: non-encoded entry location", full, ErrUnsupportedFormat) + } + e, err := decodeEntry(encoded, int(loc)) + if err != nil { + return nil, fmt.Errorf("entry %q: %w", full, err) + } + e.Path = full + entries = append(entries, e) + } + } + if c.err != nil { + return nil, fmt.Errorf("directory index: %w", c.err) + } + return entries, nil +} + +// decodeEntry decodes one bit-packed FPakEntry from the encoded blob. +// +// The leading uint32 packs: bit31 offset-is-32-bit, bit30 uncompressed-size- +// is-32-bit, bit29 size-is-32-bit, bits28-23 CompressionMethodIndex, bit22 +// encrypted, bits21-6 compression block count, bits5-0 CompressionBlockSize>>11 +// (0x3f = escape, an explicit uint32 follows). Fields then appear in this +// order: [CompressionBlockSize] Offset, UncompressedSize, [Size], [block +// sizes]. Size is omitted for stored entries (it equals UncompressedSize), and +// the per-block size table is omitted for a lone unencrypted block. +// +// The block-size-before-Offset ordering is easy to get wrong; it was pinned +// down empirically and this decoder reproduces all 173,078 records across a +// real install exactly. See docs/plans/icarus-pak-format-findings.md. +func decodeEntry(b []byte, at int) (readerEntry, error) { + c := &cursor{b: b, pos: at} + flags := c.u32() + var ( + method = int32((flags >> 23) & 0x3F) + blockCount = int((flags >> 6) & 0xFFFF) + encrypted = flags&(1<<22) != 0 + ) + if flags&0x3F == 0x3F { + c.u32() // explicit CompressionBlockSize + } + read := func(is32 bool) int64 { + if is32 { + return int64(c.u32()) + } + return int64(c.u64()) + } + offset := read(flags&(1<<31) != 0) + uncompressed := read(flags&(1<<30) != 0) + if method != 0 { + read(flags&(1<<29) != 0) // Size on disk; unused, we refuse to read these payloads + } + if blockCount > 0 && (blockCount > 1 || encrypted) { + c.bytes(4 * blockCount) + } + if c.err != nil { + return readerEntry{}, fmt.Errorf("decoding entry at blob offset %d: %w", at, c.err) + } + if encrypted { + return readerEntry{}, fmt.Errorf("%w: encrypted entry", ErrUnsupportedFormat) + } + return readerEntry{ + FileEntry: FileEntry{Size: uncompressed}, + offset: offset, + method: method, + }, nil +} + +// cursor is a bounds-checked little-endian cursor over an in-memory index +// region. It latches the first error so parse code can read a whole structure +// and check once, rather than wrapping every field. +type cursor struct { + b []byte + pos int + err error +} + +func (c *cursor) take(n int) []byte { + if c.err != nil { + return make([]byte, n) + } + if n < 0 || c.pos+n > len(c.b) { + c.err = io.ErrUnexpectedEOF + return make([]byte, max(n, 0)) + } + v := c.b[c.pos : c.pos+n] + c.pos += n + return v +} + +func (c *cursor) bytes(n int) []byte { return c.take(n) } +func (c *cursor) u32() uint32 { return binary.LittleEndian.Uint32(c.take(4)) } +func (c *cursor) i32() int32 { return int32(c.u32()) } +func (c *cursor) u64() uint64 { return binary.LittleEndian.Uint64(c.take(8)) } +func (c *cursor) i64() int64 { return int64(c.u64()) } + +// fstring reads a length-prefixed Unreal FString. A negative length signals +// UTF-16, which no pak in a real Icarus install uses and this package does not +// decode. +func (c *cursor) fstring() string { + n := c.i32() + if n == 0 || c.err != nil { + return "" + } + if n < 0 { + c.err = fmt.Errorf("%w: UTF-16 FString", ErrUnsupportedFormat) + return "" + } + return string(bytes.TrimRight(c.take(int(n)), "\x00")) +} +``` + +`reader.go` needs `bytes`, `crypto/sha1`, `encoding/binary`, `fmt`, `io`, `os`, `sort` +and `strings`. + +**Design note — enumeration vs. reading compressed entries.** `Open`/`Files` enumerate +_every_ entry regardless of compression method, and only `ReadFile` (Task 3) refuses a +non-stored payload. This is deliberate, and is what makes the "open the real pakchunk0 and +enumerate 9295 entries" acceptance step possible at all: 74% of the entries in a real +install are Oodle-compressed, so rejecting them at index-parse time would make the reader +unable to open any real pak. It does not weaken the Global Constraints — no caller can +ever obtain wrong bytes, because the refusal happens at exactly the point where wrong +bytes would otherwise be produced. + +- [ ] **Step 5: Run tests to verify they pass** + +```bash +go test ./internal/unrealpak/... -v +``` + +Expected: PASS for all five — `TestReader_Open_ListsFiles`, `TestReader_Open_RootLevelFile`, +`TestReader_Open_RejectsEncryptedIndex`, `TestReader_Open_RejectsPreVersion10` and +`TestReader_Open_RejectsCorruptedDirectoryIndex`. + +- [ ] **Step 5b: Sanity-check against the real install (manual, not committed)** + +The fixture only proves the reader agrees with itself. Point it at the real thing once: + +```bash +go run ./internal/unrealpak/... 2>/dev/null # or a throwaway main/test that calls Open+Files +``` + +Expected: `Open` succeeds on +`/data/SteamLibrary/steamapps/common/Icarus/Icarus/Content/Paks/pakchunk0-WindowsNoEditor.pak` +and `Files()` returns **9295** entries, and on +`Icarus/Content/Data/data.pak` returning **298** entries, all `.json`. If either count is +off, the index parser is wrong — fix it before Task 3. (Also listed as a post-plan +validation step.) + +- [ ] **Step 6: Commit** + +```bash +git add internal/unrealpak/pak.go internal/unrealpak/reader.go internal/unrealpak/reader_test.go +git commit -m "feat: add unrealpak footer+index reader (#136)" +``` + +--- + +## Task 3: `internal/unrealpak` — file content reader + +**Files:** + +- Modify: `internal/unrealpak/reader.go` +- Modify: `internal/unrealpak/reader_test.go` + +**Interfaces:** + +- Consumes: `Reader.entries []readerEntry` from Task 2. +- Produces: `func (r *Reader) ReadFile(path string) ([]byte, error)` — Task 12 depends on this exact signature. + +- [ ] **Step 1: Write the failing test** + +```go +func TestReader_ReadFile(t *testing.T) { + content := []byte(`{"hello":"world"}`) + path := writeMinimalPak(t, "Icarus/Content/Data/Test.json", content) + + r, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() + + got, err := r.ReadFile("Icarus/Content/Data/Test.json") + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(got) != string(content) { + t.Errorf("got %q, want %q", got, content) + } + + if _, err := r.ReadFile("does/not/exist.json"); err == nil { + t.Error("expected error for missing file, got nil") + } +} +``` + +- [ ] **Step 2: Run to verify it fails** + +```bash +go test ./internal/unrealpak/... -run TestReader_ReadFile -v +``` + +Expected: FAIL (`ReadFile` undefined). + +- [ ] **Step 3: Implement `ReadFile`** + +Add to `reader.go`: + +```go +// ReadFile returns the bytes of the entry at mount-relative path. +// +// On-disk entry data is preceded by a full FPakEntry header — 53 bytes for a +// stored entry (Offset, Size, UncompressedSize, CompressionMethodIndex, Hash, +// Flags, CompressionBlockSize) — and the index's offset points at that header, +// not the payload. The header is re-read and cross-checked rather than trusted: +// its method and size must agree with the index, and its Hash must match the +// payload's SHA1. Real paks satisfy all three (verified across a whole install), +// so a disagreement means corruption or a layout this package misread. +func (r *Reader) ReadFile(path string) ([]byte, error) { + for _, e := range r.entries { + if e.Path != path { + continue + } + // Compression is refused here rather than at index-parse time so that + // Files() can still enumerate real paks, most of whose entries are + // Oodle-compressed. No caller can obtain wrong bytes either way. + if e.method != 0 { + return nil, fmt.Errorf("unrealpak: %s: %w: compressed entry (method %d)", + path, ErrUnsupportedFormat, e.method) + } + hdr := make([]byte, storedHeaderSize) + if _, err := r.f.ReadAt(hdr, e.offset); err != nil { + return nil, fmt.Errorf("unrealpak: %s: reading entry header: %w", path, err) + } + if m := int32(binary.LittleEndian.Uint32(hdr[24:28])); m != 0 { + return nil, fmt.Errorf("unrealpak: %s: %w: compressed entry data (method %d)", + path, ErrUnsupportedFormat, m) + } + if size := int64(binary.LittleEndian.Uint64(hdr[8:16])); size != e.Size { + return nil, fmt.Errorf("unrealpak: %s: entry header size %d disagrees with index size %d", + path, size, e.Size) + } + buf := make([]byte, e.Size) + if _, err := r.f.ReadAt(buf, e.offset+storedHeaderSize); err != nil { + return nil, fmt.Errorf("unrealpak: reading %s: %w", path, err) + } + if sum := sha1.Sum(buf); !bytes.Equal(sum[:], hdr[28:48]) { //nolint:gosec + return nil, fmt.Errorf("unrealpak: %s: content hash mismatch", path) + } + return buf, nil + } + return nil, fmt.Errorf("unrealpak: %s: %w", path, os.ErrNotExist) +} +``` + +The Task 2 fixture already writes the 53-byte header ahead of each payload (a pak without +it would not be loadable), so no fixture surgery is needed here. + +Add one more test — the case that dominates in practice, since 258 of the real +`data.pak`'s 298 JSON tables are Oodle-compressed: + +```go +func TestReader_ReadFile_RejectsCompressedEntry(t *testing.T) { + const name = "Items/D_ItemsStatic.json" + path := writeMinimalPakMethod(t, name, []byte(`{"a":1}`), 1) // 1 = Oodle + + r, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() + + // Enumeration must still work — the reader lists compressed entries. + if files := r.Files(); len(files) != 1 || files[0].Path != name { + t.Fatalf("Files() = %+v, want one entry named %q", files, name) + } + if _, err := r.ReadFile(name); !errors.Is(err, ErrUnsupportedFormat) { + t.Fatalf("ReadFile error = %v, want ErrUnsupportedFormat", err) + } +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +```bash +go test ./internal/unrealpak/... -v +``` + +Expected: PASS, including the earlier `TestReader_Open_ListsFiles`. + +- [ ] **Step 5: Commit** + +```bash +git add internal/unrealpak/reader.go internal/unrealpak/reader_test.go +git commit -m "feat: add unrealpak file content reading (#136)" +``` + +--- + +## Task 4: `internal/unrealpak` — writer + +**Files:** + +- Create: `internal/unrealpak/writer.go` +- Create: `internal/unrealpak/writer_test.go` + +**Interfaces:** + +- Consumes: `entryHeaderBytes` helper pattern from Task 3 (reimplemented inline, not exported — writer and reader tests each own their fixture code per repo convention of small focused files). +- Produces: `func Create(path string) (*Writer, error)`, `func (w *Writer) AddFile(mountPath string, data []byte) error`, `func (w *Writer) Close() error` — Task 5 and Task 12 depend on these exact names. + +- [ ] **Step 1: Write the failing test** + +```go +package unrealpak + +import ( + "bytes" + "encoding/binary" + "os" + "path/filepath" + "testing" +) + +func TestWriter_CreateAndClose_ProducesValidFooter(t *testing.T) { + path := filepath.Join(t.TempDir(), "out.pak") + w, err := Create(path) + if err != nil { + t.Fatalf("Create: %v", err) + } + if err := w.AddFile("Icarus/Content/Data/Test.json", []byte(`{"a":1}`)); err != nil { + t.Fatalf("AddFile: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("reading output: %v", err) + } + if len(data) <= footerSize { + t.Fatalf("output is %d bytes, want more than a bare %d-byte footer", len(data), footerSize) + } + ft := data[len(data)-footerSize:] + if got := binary.LittleEndian.Uint32(ft[17:21]); got != magic { + t.Errorf("footer magic = %#x, want %#x", got, magic) + } + if got := int32(binary.LittleEndian.Uint32(ft[21:25])); got != writeVersion { + t.Errorf("footer version = %d, want %d", got, writeVersion) + } + if ft[16] != 0 { + t.Errorf("bEncryptedIndex = %d, want 0", ft[16]) + } +} + +// Output must not depend on AddFile ordering — Close sorts by path. +func TestWriter_Close_IsDeterministic(t *testing.T) { + build := func(order []string) []byte { + t.Helper() + path := filepath.Join(t.TempDir(), "out.pak") + w, err := Create(path) + if err != nil { + t.Fatalf("Create: %v", err) + } + for _, name := range order { + if err := w.AddFile(name, []byte(name)); err != nil { + t.Fatalf("AddFile(%s): %v", name, err) + } + } + if err := w.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("reading output: %v", err) + } + return data + } + + a := build([]string{"a/one.json", "b/two.json", "root.json"}) + b := build([]string{"root.json", "b/two.json", "a/one.json"}) + if !bytes.Equal(a, b) { + t.Error("output differs with AddFile order; Close must be deterministic") + } +} + +func TestWriter_AddFile_RejectsDuplicatePath(t *testing.T) { + w, err := Create(filepath.Join(t.TempDir(), "out.pak")) + if err != nil { + t.Fatalf("Create: %v", err) + } + defer w.Close() //nolint:errcheck + if err := w.AddFile("x.json", []byte("{}")); err != nil { + t.Fatalf("AddFile: %v", err) + } + if err := w.AddFile("x.json", []byte("{}")); err == nil { + t.Error("expected error adding a duplicate path, got nil") + } +} + +func TestWriter_AddFile_AfterClose_Errors(t *testing.T) { + path := filepath.Join(t.TempDir(), "out.pak") + w, err := Create(path) + if err != nil { + t.Fatalf("Create: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + if err := w.AddFile("x.json", []byte("{}")); err == nil { + t.Error("expected error adding file after Close, got nil") + } +} +``` + +- [ ] **Step 2: Run to verify it fails** + +```bash +go test ./internal/unrealpak/... -run TestWriter -v +``` + +Expected: FAIL (`Create` undefined). + +- [ ] **Step 3: Implement `writer.go`** + +Emits a **faithful version-11 pak**: the 221-byte footer, the primary index, the path-hash +index and the full directory index — the same four structures, in the same byte shapes, that +Icarus's own paks use and that its engine demonstrably loads. Matching the base game's own +version rather than an earlier "simplest version 7" choice is deliberate: these paks are +loaded by Icarus's actual UE runtime, not just this package's Reader, so staying +byte-shape-identical to what the engine already loads beats hoping a simpler layout is also +accepted. + +Concretely, per the verified format (see `docs/plans/icarus-pak-format-findings.md`): + +- Each entry's payload is preceded by the 53-byte stored `FPakEntry` header, `Offset` field + zeroed (real paks do the same) and `Hash` set to the payload's SHA1. +- Each index record is the 12-byte stored encoded shape — flags `0xE0000000`, `uint32` + offset, `uint32` size — byte-identical to the 4089 stored records in the real pakchunk0. +- The path-hash index uses the verified FNV-1a-64 recipe via `hashPath`, followed by an + **empty** pruned directory index (`int32(0)`), the shape 33 of 34 real paks ship. +- Entries are packed contiguously with no alignment padding, which real paks do for 8385 of + their 9294 adjacent pairs. + +```go +package unrealpak + +import ( + "bytes" + "crypto/sha1" //nolint:gosec // pak format uses SHA1, not our choice + "encoding/binary" + "fmt" + "math" + "os" + "slices" + "sort" + "strings" +) + +// writerSeed is the PathHashSeed stamped into written paks. Any value works — +// readers take the seed from the index, and real paks use a different one per +// chunk — but a fixed one keeps output deterministic. +const writerSeed uint64 = 0x9E3779B97F4A7C15 + +// Writer produces a stored (uncompressed), unencrypted version-11 pak carrying +// the full three-part index: primary index, path-hash index and full directory +// index, then the 221-byte footer. +// +// AddFile buffers content in memory and Close emits everything sorted by path, +// so identical inputs produce byte-identical output regardless of AddFile call +// order. Mod paks are small — Icarus's entire base data.pak is 2.4 MB — so +// buffering costs little, and deterministic output is worth more: it makes the +// round-trip test able to assert on bytes and keeps compiled paks stable across +// recompiles. +type Writer struct { + f *os.File + closed bool + files []writerFile + seen map[string]bool +} + +type writerFile struct { + path string + data []byte +} + +// Create opens path for writing. Call AddFile for each entry, then Close. +func Create(path string) (*Writer, error) { + f, err := os.Create(path) + if err != nil { + return nil, fmt.Errorf("unrealpak: creating %s: %w", path, err) + } + return &Writer{f: f, seen: make(map[string]bool)}, nil +} + +// AddFile records one entry. Nothing reaches disk until Close. +func (w *Writer) AddFile(mountPath string, data []byte) error { + if w.closed { + return fmt.Errorf("unrealpak: AddFile on closed writer") + } + // Root-level files are keyed under "/" in the directory index, but the + // canonical path — and the one hashPath consumes — carries no leading slash. + rel := strings.TrimPrefix(mountPath, "/") + if rel == "" { + return fmt.Errorf("unrealpak: AddFile: empty mount path") + } + if w.seen[rel] { + return fmt.Errorf("unrealpak: AddFile: duplicate path %q", rel) + } + w.seen[rel] = true + w.files = append(w.files, writerFile{path: rel, data: slices.Clone(data)}) + return nil +} + +// Close assembles the data section and all three index structures, writes them +// with the footer, and closes the file. +func (w *Writer) Close() error { + if w.closed { + return nil + } + w.closed = true + + sort.Slice(w.files, func(i, j int) bool { return w.files[i].path < w.files[j].path }) + + // Data section and encoded index records, in one pass. Each payload is + // preceded by its 53-byte header; entries are packed with no padding. + var data, encoded bytes.Buffer + locations := make(map[string]int32, len(w.files)) + for _, file := range w.files { + offset, size := int64(data.Len()), int64(len(file.data)) + if offset > math.MaxUint32 || size > math.MaxUint32 { + w.f.Close() //nolint:errcheck + return fmt.Errorf("unrealpak: %s: offset/size exceeds the 32-bit encoded-entry form this writer emits", file.path) + } + data.Write(storedEntryHeader(size, file.data)) + data.Write(file.data) + + locations[file.path] = int32(encoded.Len()) + // The 12-byte stored record: offset/uncompressed-size/size all + // 32-bit-safe, method 0, no compression blocks. + binary.Write(&encoded, binary.LittleEndian, uint32(0xE0000000)) //nolint:errcheck + binary.Write(&encoded, binary.LittleEndian, uint32(offset)) //nolint:errcheck + binary.Write(&encoded, binary.LittleEndian, uint32(size)) //nolint:errcheck + } + + // Full directory index: directory -> file -> encoded-record location. + byDir := make(map[string][]string) + for _, file := range w.files { + dir, name := splitMountPath(file.path) + byDir[dir] = append(byDir[dir], name) + } + dirNames := make([]string, 0, len(byDir)) + for dir := range byDir { + dirNames = append(dirNames, dir) + } + sort.Strings(dirNames) + + var fdi bytes.Buffer + binary.Write(&fdi, binary.LittleEndian, int32(len(dirNames))) //nolint:errcheck + for _, dir := range dirNames { + writeFString(&fdi, dir) + names := byDir[dir] + sort.Strings(names) + binary.Write(&fdi, binary.LittleEndian, int32(len(names))) //nolint:errcheck + for _, name := range names { + writeFString(&fdi, name) + binary.Write(&fdi, binary.LittleEndian, locations[strings.TrimPrefix(dir+name, "/")]) //nolint:errcheck + } + } + + // Path-hash index, then an empty pruned directory index. + var phi bytes.Buffer + binary.Write(&phi, binary.LittleEndian, int32(len(w.files))) //nolint:errcheck + for _, file := range w.files { + binary.Write(&phi, binary.LittleEndian, hashPath(file.path, writerSeed)) //nolint:errcheck + binary.Write(&phi, binary.LittleEndian, locations[file.path]) //nolint:errcheck + } + binary.Write(&phi, binary.LittleEndian, int32(0)) //nolint:errcheck // pruned index: 0 directories + + // The primary index records absolute offsets of the two sub-indexes that + // follow it; its own length is independent of those values, so measure it + // with zeros first, then rebuild with the real offsets. + phiHash := sha1.Sum(phi.Bytes()) //nolint:gosec + fdiHash := sha1.Sum(fdi.Bytes()) //nolint:gosec + count := int32(len(w.files)) + indexOffset := int64(data.Len()) + sizing := buildPrimaryIndex(count, writerSeed, 0, 0, phiHash, 0, 0, fdiHash, encoded.Bytes()) + phiOffset := indexOffset + int64(len(sizing)) + fdiOffset := phiOffset + int64(phi.Len()) + index := buildPrimaryIndex(count, writerSeed, + phiOffset, int64(phi.Len()), phiHash, + fdiOffset, int64(fdi.Len()), fdiHash, encoded.Bytes()) + indexHash := sha1.Sum(index) //nolint:gosec + + // Regions tile the file exactly, as they do in every real pak: + // data | primary index | path-hash index | full directory index | footer. + for _, chunk := range [][]byte{ + data.Bytes(), index, phi.Bytes(), fdi.Bytes(), + buildFooter(writeVersion, indexOffset, int64(len(index)), indexHash), + } { + if _, err := w.f.Write(chunk); err != nil { + w.f.Close() //nolint:errcheck + return fmt.Errorf("unrealpak: writing pak: %w", err) + } + } + return w.f.Close() +} +``` + +`writeFString`, `splitMountPath`, `storedEntryHeader`, `buildPrimaryIndex`, `buildFooter` +and `hashPath` all already exist in `pak.go` (Task 2, Step 1) — no reimplementation needed. + +- [ ] **Step 4: Run tests to verify they pass** + +```bash +go test ./internal/unrealpak/... -v +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add internal/unrealpak/writer.go internal/unrealpak/writer_test.go +git commit -m "feat: add unrealpak writer (#136)" +``` + +--- + +## Task 5: `internal/unrealpak` — round-trip integration test + +**Files:** + +- Create: `internal/unrealpak/roundtrip_test.go` + +**Interfaces:** + +- Consumes: `Create`/`AddFile`/`Close` (Task 4), `Open`/`Files`/`ReadFile` (Tasks 2–3). +- Produces: nothing new — this is the acceptance gate proving the two halves agree on format, independent of any real game file. + +- [ ] **Step 1: Write the round-trip test** + +```go +package unrealpak + +import ( + "crypto/sha1" //nolint:gosec // pak format uses SHA1, not our choice + "encoding/binary" + "os" + "path/filepath" + "testing" +) + +func TestRoundTrip_WriteThenRead(t *testing.T) { + path := filepath.Join(t.TempDir(), "roundtrip.pak") + files := map[string][]byte{ + "Icarus/Content/Data/AI-D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":235}}`), + "Icarus/Content/Data/Other.json": []byte(`{"foo":"bar"}`), + "DataTableMetadata.json": []byte(`{"root":true}`), // root-level: "/" directory key + } + + w, err := Create(path) + if err != nil { + t.Fatalf("Create: %v", err) + } + for name, data := range files { + if err := w.AddFile(name, data); err != nil { + t.Fatalf("AddFile(%s): %v", name, err) + } + } + if err := w.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + r, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() + + got := r.Files() + if len(got) != len(files) { + t.Fatalf("got %d files, want %d", len(got), len(files)) + } + for name, want := range files { + data, err := r.ReadFile(name) + if err != nil { + t.Fatalf("ReadFile(%s): %v", name, err) + } + if string(data) != string(want) { + t.Errorf("ReadFile(%s) = %q, want %q", name, data, want) + } + } +} + +// Structural assertions on the bytes themselves. Open() proves the Reader +// accepts what the Writer emits, but the Reader is not the audience that +// matters most — Icarus's engine is. These check the properties every real pak +// exhibits, so a drift away from the engine-proven shape fails here rather than +// silently in-game. +func TestRoundTrip_StructuralShape(t *testing.T) { + path := filepath.Join(t.TempDir(), "shape.pak") + w, err := Create(path) + if err != nil { + t.Fatalf("Create: %v", err) + } + content := []byte(`{"a":1}`) + if err := w.AddFile("Icarus/Content/Data/Test.json", content); err != nil { + t.Fatalf("AddFile: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("reading output: %v", err) + } + ft := data[len(data)-footerSize:] + indexOffset := int64(binary.LittleEndian.Uint64(ft[25:33])) + indexSize := int64(binary.LittleEndian.Uint64(ft[33:41])) + + // The footer's SHA1 must cover the primary index exactly. + indexSum := sha1.Sum(data[indexOffset : indexOffset+indexSize]) //nolint:gosec + if string(indexSum[:]) != string(ft[41:61]) { + t.Error("footer IndexHash does not match the primary index bytes") + } + + // The data section holds one 53-byte header plus the payload, and the + // index starts immediately after it — regions tile with no gap. + if want := int64(storedHeaderSize + len(content)); indexOffset != want { + t.Errorf("index starts at %d, want %d (53-byte header + %d-byte payload)", + indexOffset, want, len(content)) + } + // The per-entry header's Hash field must be the payload's SHA1. + if sum := sha1.Sum(content); string(sum[:]) != string(data[28:48]) { //nolint:gosec + t.Error("per-entry header Hash does not match the payload SHA1") + } + // Its Offset field is zero, as in every real pak. + if got := binary.LittleEndian.Uint64(data[0:8]); got != 0 { + t.Errorf("per-entry header Offset = %d, want 0", got) + } +} +``` + +- [ ] **Step 2: Run to verify it passes** + +```bash +go test ./internal/unrealpak/... -v +``` + +Expected: PASS. If it fails, the Writer and Reader disagree on layout — fix before proceeding to any Icarus-specific code, since everything downstream depends on this package being internally consistent. + +- [ ] **Step 3: Commit** + +```bash +git add internal/unrealpak/roundtrip_test.go +git commit -m "test: add unrealpak writer/reader round-trip coverage (#136)" +``` + +--- + +## Task 6: Icarus source — Firestore typed-value decoder + +**Files:** + +- Create: `internal/source/icarus/firestore_value.go` +- Create: `internal/source/icarus/firestore_value_test.go` + +**Interfaces:** + +- Produces: `func decodeFields(fields map[string]any) map[string]any` — Task 8's mapping code depends on this exact name/signature. + +- [ ] **Step 1: Write the failing test** + +```go +package icarus + +import ( + "reflect" + "testing" +) + +func TestDecodeFields(t *testing.T) { + // Shape of a real Firestore REST document's "fields" object. + raw := map[string]any{ + "name": map[string]any{"stringValue": "Bear Mount"}, + "version": map[string]any{"stringValue": "3.3"}, + "files": map[string]any{"mapValue": map[string]any{"fields": map[string]any{ + "pak": map[string]any{"stringValue": "https://example.com/mod.pak"}, + "exmodz": map[string]any{"stringValue": "https://example.com/mod.exmodz"}, + }}}, + "missing": map[string]any{"nullValue": nil}, + } + + got := decodeFields(raw) + + want := map[string]any{ + "name": "Bear Mount", + "version": "3.3", + "files": map[string]any{ + "pak": "https://example.com/mod.pak", + "exmodz": "https://example.com/mod.exmodz", + }, + "missing": nil, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("decodeFields() = %#v, want %#v", got, want) + } +} +``` + +- [ ] **Step 2: Run to verify it fails** + +```bash +go test ./internal/source/icarus/... -run TestDecodeFields -v +``` + +Expected: FAIL (`decodeFields` undefined, package may not exist yet — create the directory as part of this step). + +- [ ] **Step 3: Implement** + +```go +package icarus + +// decodeFields unwraps a Firestore REST document's typed-value "fields" +// object (each value wrapped as {"stringValue": ...} / {"mapValue": {...}} / +// etc.) into plain Go values. Only the value kinds this catalog's schema +// actually uses are handled; anything else decodes to nil rather than +// panicking, since an unrecognized field should be ignorable, not fatal. +func decodeFields(fields map[string]any) map[string]any { + out := make(map[string]any, len(fields)) + for k, v := range fields { + out[k] = decodeValue(v) + } + return out +} + +func decodeValue(v any) any { + wrapped, ok := v.(map[string]any) + if !ok { + return nil + } + if s, ok := wrapped["stringValue"]; ok { + return s + } + if b, ok := wrapped["booleanValue"]; ok { + return b + } + if i, ok := wrapped["integerValue"]; ok { + return i + } + if d, ok := wrapped["doubleValue"]; ok { + return d + } + if m, ok := wrapped["mapValue"]; ok { + mv, _ := m.(map[string]any) + inner, _ := mv["fields"].(map[string]any) + return decodeFields(inner) + } + if a, ok := wrapped["arrayValue"]; ok { + av, _ := a.(map[string]any) + values, _ := av["values"].([]any) + out := make([]any, len(values)) + for i, item := range values { + out[i] = decodeValue(item) + } + return out + } + if _, ok := wrapped["nullValue"]; ok { + return nil + } + return nil +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +```bash +go test ./internal/source/icarus/... -v +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add internal/source/icarus/firestore_value.go internal/source/icarus/firestore_value_test.go +git commit -m "feat: add Firestore typed-value decoder for Icarus source (#136)" +``` + +--- + +## Task 7: Icarus source — Firestore REST client + +**Files:** + +- Create: `internal/source/icarus/firestore_client.go` +- Create: `internal/source/icarus/firestore_client_test.go` + +**Interfaces:** + +- Consumes: `decodeFields` (Task 6). +- Produces: `type firestoreDoc struct { ID string; Fields map[string]any }`, `func (c *firestoreClient) listCollection(ctx context.Context, collection string) ([]firestoreDoc, error)`, `func (c *firestoreClient) getDocument(ctx context.Context, collection, docID string) (*firestoreDoc, error)`, `func newFirestoreClient(projectID string, httpClient *http.Client) *firestoreClient` — Task 8 depends on all four names. + +- [ ] **Step 1: Write the failing test** + +```go +package icarus + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestFirestoreClient_ListCollection_Paginates(t *testing.T) { + pages := []map[string]any{ + { + "documents": []map[string]any{ + {"name": "projects/p/databases/(default)/documents/mods/abc", "fields": map[string]any{"name": map[string]any{"stringValue": "Bear Mount"}}}, + }, + "nextPageToken": "page2", + }, + { + "documents": []map[string]any{ + {"name": "projects/p/databases/(default)/documents/mods/def", "fields": map[string]any{"name": map[string]any{"stringValue": "Wolf Mount"}}}, + }, + }, + } + callCount := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + page := pages[callCount] + callCount++ + json.NewEncoder(w).Encode(page) //nolint:errcheck + })) + defer srv.Close() + + c := newFirestoreClient("test-project", srv.Client()) + c.baseURL = srv.URL // test seam, see Step 3 + + docs, err := c.listCollection(context.Background(), "mods") + if err != nil { + t.Fatalf("listCollection: %v", err) + } + if len(docs) != 2 { + t.Fatalf("got %d docs, want 2 (pagination should have followed nextPageToken)", len(docs)) + } + if docs[0].ID != "abc" || docs[1].ID != "def" { + t.Errorf("doc IDs = %q, %q, want abc, def", docs[0].ID, docs[1].ID) + } + if docs[0].Fields["name"] != "Bear Mount" { + t.Errorf("docs[0].Fields[name] = %v, want Bear Mount", docs[0].Fields["name"]) + } + if callCount != 2 { + t.Errorf("callCount = %d, want 2 (one per page)", callCount) + } +} + +func TestFirestoreClient_GetDocument_NotFound(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + c := newFirestoreClient("test-project", srv.Client()) + c.baseURL = srv.URL + + _, err := c.getDocument(context.Background(), "mods", "missing") + if err == nil { + t.Fatal("expected error for 404, got nil") + } +} +``` + +- [ ] **Step 2: Run to verify it fails** + +```bash +go test ./internal/source/icarus/... -run TestFirestoreClient -v +``` + +Expected: FAIL (`newFirestoreClient` undefined). + +- [ ] **Step 3: Implement** + +```go +package icarus + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" +) + +const defaultFirestoreBaseURL = "https://firestore.googleapis.com/v1" + +// firestoreDoc is a decoded Firestore document: ID is the last path segment +// of its resource name, Fields is already unwrapped via decodeFields. +type firestoreDoc struct { + ID string + Fields map[string]any +} + +type firestoreClient struct { + projectID string + httpClient *http.Client + baseURL string // overridable in tests; defaults to defaultFirestoreBaseURL +} + +func newFirestoreClient(projectID string, httpClient *http.Client) *firestoreClient { + if httpClient == nil { + httpClient = http.DefaultClient + } + return &firestoreClient{projectID: projectID, httpClient: httpClient, baseURL: defaultFirestoreBaseURL} +} + +func (c *firestoreClient) documentsURL() string { + return fmt.Sprintf("%s/projects/%s/databases/(default)/documents", c.baseURL, c.projectID) +} + +// listCollection fetches every document in collection, following +// nextPageToken until exhausted (the catalog reads Firestore unauthenticated +// and public, with no server-side query support in play — see the design +// doc's "fetch-all + filter client-side" decision). +func (c *firestoreClient) listCollection(ctx context.Context, collection string) ([]firestoreDoc, error) { + var all []firestoreDoc + pageToken := "" + for { + url := fmt.Sprintf("%s/%s?pageSize=200", c.documentsURL(), collection) + if pageToken != "" { + url += "&pageToken=" + pageToken + } + var page struct { + Documents []struct { + Name string `json:"name"` + Fields map[string]any `json:"fields"` + } `json:"documents"` + NextPageToken string `json:"nextPageToken"` + } + if err := c.getJSON(ctx, url, &page); err != nil { + return nil, fmt.Errorf("listing %s: %w", collection, err) + } + for _, d := range page.Documents { + all = append(all, firestoreDoc{ID: lastPathSegment(d.Name), Fields: decodeFields(d.Fields)}) + } + if page.NextPageToken == "" { + break + } + pageToken = page.NextPageToken + } + return all, nil +} + +// getDocument fetches a single document by ID. +func (c *firestoreClient) getDocument(ctx context.Context, collection, docID string) (*firestoreDoc, error) { + url := fmt.Sprintf("%s/%s/%s", c.documentsURL(), collection, docID) + var doc struct { + Name string `json:"name"` + Fields map[string]any `json:"fields"` + } + if err := c.getJSON(ctx, url, &doc); err != nil { + return nil, fmt.Errorf("fetching %s/%s: %w", collection, docID, err) + } + return &firestoreDoc{ID: lastPathSegment(doc.Name), Fields: decodeFields(doc.Fields)}, nil +} + +func (c *firestoreClient) getJSON(ctx context.Context, url string, out any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + resp, err := c.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() //nolint:errcheck + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("HTTP %d", resp.StatusCode) + } + return json.NewDecoder(resp.Body).Decode(out) +} + +func lastPathSegment(resourceName string) string { + parts := strings.Split(resourceName, "/") + return parts[len(parts)-1] +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +```bash +go test ./internal/source/icarus/... -v +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add internal/source/icarus/firestore_client.go internal/source/icarus/firestore_client_test.go +git commit -m "feat: add Firestore REST client for Icarus source (#136)" +``` + +--- + +## Task 8: Icarus source — `ModSource` implementation + +**Files:** + +- Create: `internal/source/icarus/icarus.go` +- Create: `internal/source/icarus/icarus_test.go` + +**Interfaces:** + +- Consumes: `newFirestoreClient`, `listCollection`, `getDocument` (Task 7); `source.ModSource`, `source.SearchQuery`, `source.SearchResult`, `source.ErrNotSupported`, `domain.Mod`, `domain.DownloadableFile`, `domain.ModReference`, `domain.InstalledMod`, `domain.Update` (existing repo types, confirmed above). +- Produces: `func New(httpClient *http.Client, projectID string) *Icarus`, satisfying `source.ModSource` and `source.CapabilityReporter` — Task 9 depends on this constructor signature. + +- [ ] **Step 1: Write the failing test** + +```go +package icarus + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/source" +) + +func modsListHandler(mods []map[string]any) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + docs := make([]map[string]any, len(mods)) + for i, m := range mods { + docs[i] = map[string]any{ + "name": "projects/p/databases/(default)/documents/mods/" + m["id"].(string), + "fields": m["fields"], + } + } + json.NewEncoder(w).Encode(map[string]any{"documents": docs}) //nolint:errcheck + } +} + +func TestIcarus_Search_FiltersClientSide(t *testing.T) { + srv := httptest.NewServer(modsListHandler([]map[string]any{ + {"id": "abc", "fields": map[string]any{ + "name": map[string]any{"stringValue": "Bear Mount"}, "author": map[string]any{"stringValue": "Jimk72"}, + "description": map[string]any{"stringValue": "Ride a bear"}, "version": map[string]any{"stringValue": "3.3"}, + "compatibility": map[string]any{"stringValue": "w57"}, + "files": map[string]any{"mapValue": map[string]any{"fields": map[string]any{"exmodz": map[string]any{"stringValue": "https://x/bear.exmodz"}}}}, + }}, + {"id": "def", "fields": map[string]any{ + "name": map[string]any{"stringValue": "Wolf Pack"}, "author": map[string]any{"stringValue": "Someone"}, + "description": map[string]any{"stringValue": "Tame wolves"}, "version": map[string]any{"stringValue": "1.0"}, + "files": map[string]any{"mapValue": map[string]any{"fields": map[string]any{"pak": map[string]any{"stringValue": "https://x/wolf.pak"}}}}, + }}, + })) + defer srv.Close() + + src := New(srv.Client(), "test-project") + src.firestore.baseURL = srv.URL + + result, err := src.Search(context.Background(), source.SearchQuery{Query: "bear"}) + if err != nil { + t.Fatalf("Search: %v", err) + } + if len(result.Mods) != 1 || result.Mods[0].Name != "Bear Mount" { + t.Fatalf("Search(%q) = %+v, want exactly Bear Mount", "bear", result.Mods) + } + if result.Mods[0].GameID != "icarus" { + t.Errorf("GameID = %q, want icarus", result.Mods[0].GameID) + } +} + +func TestIcarus_GetModFiles_ReturnsExmodzAndPak(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]any{ //nolint:errcheck + "name": "projects/p/databases/(default)/documents/mods/abc", + "fields": map[string]any{ + "name": map[string]any{"stringValue": "Bear Mount"}, + "files": map[string]any{"mapValue": map[string]any{"fields": map[string]any{ + "exmodz": map[string]any{"stringValue": "https://x/bear.exmodz"}, + }}}, + }, + }) + })) + defer srv.Close() + + src := New(srv.Client(), "test-project") + src.firestore.baseURL = srv.URL + + files, err := src.GetModFiles(context.Background(), &domain.Mod{ID: "abc", GameID: "icarus"}) + if err != nil { + t.Fatalf("GetModFiles: %v", err) + } + if len(files) != 1 || files[0].FileName != "bear.exmodz" { + t.Fatalf("files = %+v, want one bear.exmodz entry", files) + } + if !files[0].IsPrimary { + t.Error("single file should be marked primary") + } +} +``` + +- [ ] **Step 2: Run to verify it fails** + +```bash +go test ./internal/source/icarus/... -run TestIcarus -v +``` + +Expected: FAIL (`New` undefined). + +- [ ] **Step 3: Implement `icarus.go`** + +```go +package icarus + +import ( + "context" + "fmt" + "net/http" + "net/url" + "path" + "strings" + + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/source" +) + +// gameID is fixed: the Firestore database this source reads is Icarus-only. +const gameID = "icarus" + +// Icarus is a ModSource backed by the public, unauthenticated Firestore REST +// API described in docs/plans/2026-07-29-icarus-exmod-pak-research.md. +type Icarus struct { + firestore *firestoreClient +} + +// New constructs an Icarus source. projectID is the Firestore project ID +// (from the Firebase console) — passed explicitly rather than hard-coded so +// tests can point at an httptest server and so the real value lives in one +// place at the call site (Task 9), not buried in this package. +func New(httpClient *http.Client, projectID string) *Icarus { + return &Icarus{firestore: newFirestoreClient(projectID, httpClient)} +} + +var ( + _ source.ModSource = (*Icarus)(nil) + _ source.CapabilityReporter = (*Icarus)(nil) +) + +func (s *Icarus) ID() string { return "icarus" } +func (s *Icarus) Name() string { return "Icarus (Project Daedalus)" } + +// AuthURL/ExchangeToken: unsupported — Firestore reads here are public. +func (s *Icarus) AuthURL() string { return "" } +func (s *Icarus) ExchangeToken(ctx context.Context, code string) (*source.Token, error) { + return nil, fmt.Errorf("source %q: authentication: %w", s.ID(), source.ErrNotSupported) +} + +// GetDependencies: the modinfo.json v2 schema has no dependency field. +func (s *Icarus) GetDependencies(ctx context.Context, mod *domain.Mod) ([]domain.ModReference, error) { + return nil, fmt.Errorf("source %q: dependencies: %w", s.ID(), source.ErrNotSupported) +} + +func (s *Icarus) Capabilities() source.Capabilities { + return source.Capabilities{Search: true, Dependencies: false, Updates: true, Auth: false} +} + +func (s *Icarus) TypeLabel() string { return "built-in" } + +// Search fetches the whole mods collection and filters client-side — this +// catalog has no server-side query support to speak of, matching +// project_daedalus's own ModsController#find_mods approach. +func (s *Icarus) Search(ctx context.Context, query source.SearchQuery) (source.SearchResult, error) { + docs, err := s.firestore.listCollection(ctx, "mods") + if err != nil { + return source.SearchResult{}, fmt.Errorf("source %q: searching: %w", s.ID(), err) + } + + var mods []domain.Mod + q := strings.ToLower(query.Query) + for _, d := range docs { + m := mapDoc(d) + if q == "" || strings.Contains(strings.ToLower(m.Name), q) || + strings.Contains(strings.ToLower(m.Author), q) || + strings.Contains(strings.ToLower(m.Description), q) { + mods = append(mods, m) + } + } + + pageSize := query.PageSize + if pageSize <= 0 { + pageSize = 20 + } + page := query.Page + if page < 0 { + page = 0 + } + start := page * pageSize + if start > len(mods) { + start = len(mods) + } + end := start + pageSize + if end > len(mods) { + end = len(mods) + } + + return source.SearchResult{Mods: mods[start:end], TotalCount: len(mods), Page: page, PageSize: pageSize}, nil +} + +func (s *Icarus) GetMod(ctx context.Context, queryGameID, modID string) (*domain.Mod, error) { + doc, err := s.firestore.getDocument(ctx, "mods", modID) + if err != nil { + return nil, fmt.Errorf("source %q: fetching mod %s: %w", s.ID(), modID, err) + } + m := mapDoc(*doc) + return &m, nil +} + +// GetModFiles returns the mod's downloadable files (pak and/or exmodz — see +// modinfo.json v2 schema). A single file is marked primary, matching the +// existing custom.API convention. +func (s *Icarus) GetModFiles(ctx context.Context, mod *domain.Mod) ([]domain.DownloadableFile, error) { + doc, err := s.firestore.getDocument(ctx, "mods", mod.ID) + if err != nil { + return nil, fmt.Errorf("source %q: listing files for %s: %w", s.ID(), mod.ID, err) + } + filesField, _ := doc.Fields["files"].(map[string]any) + var out []domain.DownloadableFile + for _, kind := range []string{"pak", "exmodz"} { + rawURL, ok := filesField[kind].(string) + if !ok || rawURL == "" { + continue + } + out = append(out, domain.DownloadableFile{ + ID: kind, + Name: kind, + FileName: fileNameFromURL(rawURL, kind), + Category: strings.ToUpper(kind), + }) + } + if len(out) == 1 { + out[0].IsPrimary = true + } + return out, nil +} + +// GetDownloadURL re-fetches the mod document and returns the stored URL for +// fileID ("pak" or "exmodz") directly — no signing, matching a static-URL +// catalog rather than an OAuth-gated one. +func (s *Icarus) GetDownloadURL(ctx context.Context, mod *domain.Mod, fileID string) (string, error) { + doc, err := s.firestore.getDocument(ctx, "mods", mod.ID) + if err != nil { + return "", fmt.Errorf("source %q: download URL for %s: %w", s.ID(), fileID, err) + } + filesField, _ := doc.Fields["files"].(map[string]any) + rawURL, ok := filesField[fileID].(string) + if !ok || rawURL == "" { + return "", fmt.Errorf("source %q: file %s: no download URL", s.ID(), fileID) + } + return rawURL, nil +} + +// CheckUpdates compares each installed mod's stored version against the +// catalog's current version string (semantic-ish, per modinfo.json's +// "recommended" versioning note — not guaranteed strictly semver, so this +// uses domain.IsNewerVersion the same way custom.API does). +func (s *Icarus) CheckUpdates(ctx context.Context, installed []domain.InstalledMod) ([]domain.Update, error) { + var updates []domain.Update + var errs []error + for _, inst := range installed { + select { + case <-ctx.Done(): + return updates, ctx.Err() + default: + } + current, err := s.GetMod(ctx, gameID, inst.ID) + if err != nil { + errs = append(errs, err) + continue + } + if domain.IsNewerVersion(inst.Version, current.Version) { + updates = append(updates, domain.Update{InstalledMod: inst, NewVersion: current.Version}) + } + } + if len(errs) > 0 { + return updates, fmt.Errorf("source %q: %d update check(s) failed: %v", s.ID(), len(errs), errs[0]) + } + return updates, nil +} + +// mapDoc converts a decoded Firestore document into domain.Mod per the +// modinfo.json v2 schema (docs/plans/2026-07-29-icarus-exmod-pak-research.md). +func mapDoc(d firestoreDoc) domain.Mod { + str := func(key string) string { + s, _ := d.Fields[key].(string) + return s + } + return domain.Mod{ + ID: d.ID, + SourceID: "icarus", + GameID: gameID, + Name: str("name"), + Author: str("author"), + Version: str("version"), + Category: str("compatibility"), // Icarus week-build string, e.g. "w57" + Description: str("description"), + PictureURL: str("imageURL"), + SourceURL: str("readmeURL"), + } +} + +func fileNameFromURL(rawURL, fallbackExt string) string { + u, err := url.Parse(rawURL) + if err != nil || u.Path == "" { + return fallbackExt + } + base := path.Base(u.Path) + if base == "." || base == "/" { + return fallbackExt + } + return base +} + +var _ = strconv.Itoa // silence unused import if strconv ends up unused after edits; remove if genuinely unused +``` + +(Drop the trailing `var _ = strconv.Itoa` line and the `strconv` import if `go vet`/`goimports` flags it as unused once the real file is assembled — included here only as a reminder to check, not to ship.) + +- [ ] **Step 4: Run tests to verify they pass** + +```bash +go test ./internal/source/icarus/... -v +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add internal/source/icarus/icarus.go internal/source/icarus/icarus_test.go +git commit -m "feat: implement Icarus ModSource over Firestore REST (#136)" +``` + +--- + +## Task 9: Register Icarus as a built-in source + +**Files:** + +- Modify: `cmd/lmm/root.go:213-216` (`builtinSourceFactories`) +- Modify: `README.md` (document the new source + required manual `games.yaml` entry, per this repo's "keep README updated when functional components change" convention) + +**Interfaces:** + +- Consumes: `icarus.New(httpClient *http.Client, projectID string) *Icarus` (Task 8). +- Produces: nothing new — this is wiring only. + +- [ ] **Step 1: Add the factory** + +In `cmd/lmm/root.go`, add the import and extend `builtinSourceFactories`: + +```go +import ( + // ...existing imports... + "github.com/DonovanMods/linux-mod-manager/internal/source/icarus" +) + +// icarusFirestoreProjectID is Project Daedalus's Firebase project ID, from +// the Firebase console. It is public information (Firestore reads are +// unauthenticated by design, per the research spike) — this constant is the +// one place it needs to be substituted with the real value (see "Post-plan +// manual validation" item 2 at the end of this plan). +const icarusFirestoreProjectID = "project-daedalus" + +var builtinSourceFactories = []func() source.ModSource{ + func() source.ModSource { return nexusmods.New(nil, "") }, + func() source.ModSource { return curseforge.New(nil, "") }, + func() source.ModSource { return icarus.New(nil, icarusFirestoreProjectID) }, +} +``` + +- [ ] **Step 2: Write a registration smoke test** + +Add to `cmd/lmm/root_test.go` (or wherever `builtinSourceFactories` is already exercised — check for an existing test asserting NexusMods/CurseForge register cleanly, and follow its shape): + +```go +func TestBuiltinSourceFactories_IncludesIcarus(t *testing.T) { + found := false + for _, factory := range builtinSourceFactories { + if factory().ID() == "icarus" { + found = true + } + } + if !found { + t.Error("builtinSourceFactories should include the icarus source") + } +} +``` + +- [ ] **Step 3: Run to verify it passes** + +```bash +go build ./... && go test ./cmd/lmm/... -run TestBuiltinSourceFactories -v +``` + +Expected: PASS. + +- [ ] **Step 4: Document the manual game config in README.md** + +Add an entry alongside this repo's existing per-game setup examples (find the section documenting `games.yaml` entries for existing sources and follow its exact format) showing: + +```yaml +games: + icarus: + name: Icarus + install_path: /path/to/Steam/steamapps/common/Icarus + mod_path: /path/to/Steam/steamapps/common/Icarus/Icarus/Content/Paks/mods + deploy_mode: compile # added in Task 13 + source_ids: + icarus: icarus +``` + +Note in prose: Steam auto-detection (`lmm game detect`) does not yet know about Icarus — this is a manual `games.yaml` entry for now (App ID 1149460 confirmed during the research spike; auto-detection is a separate, smaller follow-up not covered by this plan). + +- [ ] **Step 5: Commit** + +```bash +git add cmd/lmm/root.go cmd/lmm/root_test.go README.md +git commit -m "feat: register Icarus as a built-in mod source (#136)" +``` + +--- + +## Task 10: `.EXMOD` diff schema + row-patch application + +**Files:** + +- Create: `internal/source/icarus/exmod.go` +- Create: `internal/source/icarus/exmod_test.go` + +**Interfaces:** + +- Produces: `type ExmodDiff struct { Name, Author, Version, Description string; Rows []ExmodRow }`, `type ExmodRow struct { CurrentFile string; FileItems []ExmodFileItem }`, `type ExmodFileItem struct { Name string; Fields map[string]any }`, `func ParseExmod(data []byte) (*ExmodDiff, error)`, `func ApplyRowPatch(baseJSON []byte, row ExmodRow) ([]byte, error)` — Task 12 depends on all of these. + +- [ ] **Step 1: Write the failing test** + +```go +package icarus + +import ( + "encoding/json" + "testing" +) + +const sampleExmod = `{ + "name": "Bear Mount", + "author": "Jimk72", + "version": "3.3", + "description": "Allows raising cubs", + "Rows": [ + { + "CurrentFile": "AI-D_AIGrowth.json", + "File_Items": [ + {"Name": "Mount_Bear", "BaseMovementSpeed": 235, "BaseSwimSpeed": 300} + ] + } + ] +}` + +func TestParseExmod(t *testing.T) { + diff, err := ParseExmod([]byte(sampleExmod)) + if err != nil { + t.Fatalf("ParseExmod: %v", err) + } + if diff.Name != "Bear Mount" || diff.Version != "3.3" { + t.Errorf("Name/Version = %q/%q, want Bear Mount/3.3", diff.Name, diff.Version) + } + if len(diff.Rows) != 1 || diff.Rows[0].CurrentFile != "AI-D_AIGrowth.json" { + t.Fatalf("Rows = %+v", diff.Rows) + } + if len(diff.Rows[0].FileItems) != 1 || diff.Rows[0].FileItems[0].Name != "Mount_Bear" { + t.Fatalf("FileItems = %+v", diff.Rows[0].FileItems) + } + if diff.Rows[0].FileItems[0].Fields["BaseMovementSpeed"] != float64(235) { + t.Errorf("BaseMovementSpeed = %v, want 235", diff.Rows[0].FileItems[0].Fields["BaseMovementSpeed"]) + } +} + +func TestApplyRowPatch_OverwritesNamedRowFieldsOnly(t *testing.T) { + base := []byte(`{ + "Mount_Bear": {"BaseMovementSpeed": 200, "BaseSwimSpeed": 150, "Untouched": "keep-me"}, + "Other_Row": {"BaseMovementSpeed": 999} + }`) + row := ExmodRow{ + CurrentFile: "AI-D_AIGrowth.json", + FileItems: []ExmodFileItem{ + {Name: "Mount_Bear", Fields: map[string]any{"BaseMovementSpeed": float64(235)}}, + }, + } + + got, err := ApplyRowPatch(base, row) + if err != nil { + t.Fatalf("ApplyRowPatch: %v", err) + } + + var result map[string]map[string]any + if err := json.Unmarshal(got, &result); err != nil { + t.Fatalf("unmarshaling result: %v", err) + } + if result["Mount_Bear"]["BaseMovementSpeed"] != float64(235) { + t.Errorf("BaseMovementSpeed not patched: %v", result["Mount_Bear"]["BaseMovementSpeed"]) + } + if result["Mount_Bear"]["Untouched"] != "keep-me" { + t.Errorf("unrelated field was clobbered: %v", result["Mount_Bear"]["Untouched"]) + } + if result["Other_Row"]["BaseMovementSpeed"] != float64(999) { + t.Errorf("unrelated row was modified: %v", result["Other_Row"]) + } +} + +func TestApplyRowPatch_UnknownRowName_Errors(t *testing.T) { + base := []byte(`{"Mount_Bear": {}}`) + row := ExmodRow{FileItems: []ExmodFileItem{{Name: "Does_Not_Exist", Fields: map[string]any{"X": 1}}}} + + if _, err := ApplyRowPatch(base, row); err == nil { + t.Error("expected error for unknown row name (no silent fallback), got nil") + } +} +``` + +- [ ] **Step 2: Run to verify it fails** + +```bash +go test ./internal/source/icarus/... -run "TestParseExmod|TestApplyRowPatch" -v +``` + +Expected: FAIL (`ParseExmod`/`ApplyRowPatch` undefined). + +- [ ] **Step 3: Implement** + +The real sample's `File_Items` entries mix a fixed `Name` key with arbitrary game-specific override keys in the same object (see `Bear_Mount.EXMOD`, where some entries additionally nest a `Base` sub-object of `(Value="...")`-keyed stats — this plan's `ExmodFileItem.Fields` deliberately captures "everything except Name" generically, so both shapes round-trip through `json.RawMessage`/`any` without this package needing to special-case every game-data shape it might see): + +```go +package icarus + +import ( + "encoding/json" + "fmt" +) + +// ExmodDiff is the parsed .EXMOD manifest — a diff against the base game's +// JSON data tables, not a binary/compiled-asset diff (confirmed against a +// real sample; see docs/plans/2026-07-29-icarus-exmod-pak-research.md). +type ExmodDiff struct { + Name string + Author string + Version string + Description string + Rows []ExmodRow +} + +// ExmodRow targets one base data-table file (e.g. "AI-D_AIGrowth.json"). +type ExmodRow struct { + CurrentFile string + FileItems []ExmodFileItem +} + +// ExmodFileItem overrides fields on the base row named Name. Fields holds +// every key from the source JSON except "Name" itself, generically — the +// real schema nests arbitrary game-data shapes here (see package doc +// comment), so this deliberately does not enumerate them. +type ExmodFileItem struct { + Name string + Fields map[string]any +} + +func ParseExmod(data []byte) (*ExmodDiff, error) { + var raw struct { + Name string `json:"name"` + Author string `json:"author"` + Version string `json:"version"` + Description string `json:"description"` + Rows []struct { + CurrentFile string `json:"CurrentFile"` + FileItems []map[string]any `json:"File_Items"` + } `json:"Rows"` + } + if err := json.Unmarshal(data, &raw); err != nil { + return nil, fmt.Errorf("icarus: parsing .EXMOD: %w", err) + } + + diff := &ExmodDiff{Name: raw.Name, Author: raw.Author, Version: raw.Version, Description: raw.Description} + for _, r := range raw.Rows { + row := ExmodRow{CurrentFile: r.CurrentFile} + for _, item := range r.FileItems { + name, _ := item["Name"].(string) + if name == "" { + return nil, fmt.Errorf("icarus: .EXMOD row in %s: File_Items entry missing Name", r.CurrentFile) + } + fields := make(map[string]any, len(item)-1) + for k, v := range item { + if k == "Name" { + continue + } + fields[k] = v + } + row.FileItems = append(row.FileItems, ExmodFileItem{Name: name, Fields: fields}) + } + diff.Rows = append(diff.Rows, row) + } + return diff, nil +} + +// ApplyRowPatch merges row's named-row field overrides into baseJSON (a base +// game data-table file keyed by row name, e.g. {"Mount_Bear": {...}, ...}) +// and returns the patched document. Fails loudly (no silent fallback, repo +// precedent #95) if a targeted row name doesn't exist in the base — that +// means either the base version is stale relative to the mod, or the exmod +// targets a file this function was called with by mistake. +func ApplyRowPatch(baseJSON []byte, row ExmodRow) ([]byte, error) { + var doc map[string]map[string]any + if err := json.Unmarshal(baseJSON, &doc); err != nil { + return nil, fmt.Errorf("icarus: parsing base data table %s: %w", row.CurrentFile, err) + } + for _, item := range row.FileItems { + target, ok := doc[item.Name] + if !ok { + return nil, fmt.Errorf("icarus: %s: row %q not found in base data table", row.CurrentFile, item.Name) + } + for k, v := range item.Fields { + target[k] = v + } + doc[item.Name] = target + } + return json.Marshal(doc) +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +```bash +go test ./internal/source/icarus/... -v +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add internal/source/icarus/exmod.go internal/source/icarus/exmod_test.go +git commit -m "feat: add .EXMOD parsing and row-patch application (#136)" +``` + +--- + +## Task 11: `.EXMODZ` archive unpacking + +**Files:** + +- Create: `internal/source/icarus/exmodz.go` +- Create: `internal/source/icarus/exmodz_test.go` + +**Interfaces:** + +- Consumes: `ParseExmod` (Task 10). +- Produces: `type ExmodzBundle struct { Diff *ExmodDiff; Assets map[string][]byte }`, `func ParseExmodz(zipData []byte) (*ExmodzBundle, error)` — Task 12 depends on this. + +- [ ] **Step 1: Write the failing test** + +```go +package icarus + +import ( + "archive/zip" + "bytes" + "testing" +) + +// buildTestExmodz mirrors the real Bear_Mount.EXMODZ layout: a manifest +// under "Extracted Mods/.EXMOD" plus loose asset files at paths that +// mirror in-game mount structure. +func buildTestExmodz(t *testing.T) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + + manifest := `{"name":"Bear Mount","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":235}]}]}` + w, err := zw.Create("Extracted Mods/Bear_Mount.EXMOD") + if err != nil { + t.Fatal(err) + } + w.Write([]byte(manifest)) //nolint:errcheck + + assetW, err := zw.Create("Bear_Mount/ASS/ITM/SK_ITM_Saddle_Bear.uasset") + if err != nil { + t.Fatal(err) + } + assetW.Write([]byte("fake-uasset-bytes")) //nolint:errcheck + + if err := zw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func TestParseExmodz(t *testing.T) { + bundle, err := ParseExmodz(buildTestExmodz(t)) + if err != nil { + t.Fatalf("ParseExmodz: %v", err) + } + if bundle.Diff == nil || bundle.Diff.Name != "Bear Mount" { + t.Fatalf("Diff = %+v", bundle.Diff) + } + asset, ok := bundle.Assets["Bear_Mount/ASS/ITM/SK_ITM_Saddle_Bear.uasset"] + if !ok { + t.Fatalf("Assets missing expected key; got keys: %v", mapKeys(bundle.Assets)) + } + if string(asset) != "fake-uasset-bytes" { + t.Errorf("asset content = %q", asset) + } +} + +func mapKeys(m map[string][]byte) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} + +func TestParseExmodz_NoManifest_Errors(t *testing.T) { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, _ := zw.Create("readme.txt") + w.Write([]byte("no manifest here")) //nolint:errcheck + zw.Close() //nolint:errcheck + + if _, err := ParseExmodz(buf.Bytes()); err == nil { + t.Error("expected error when no .EXMOD manifest is present, got nil") + } +} +``` + +- [ ] **Step 2: Run to verify it fails** + +```bash +go test ./internal/source/icarus/... -run TestParseExmodz -v +``` + +Expected: FAIL (`ParseExmodz` undefined). + +- [ ] **Step 3: Implement** + +```go +package icarus + +import ( + "archive/zip" + "bytes" + "fmt" + "io" + "strings" +) + +// ExmodzBundle is a parsed .EXMODZ: the diff manifest plus any pre-built +// asset files the mod author already compiled (placed as-is into the output +// pak — never recompiled by LMM). +type ExmodzBundle struct { + Diff *ExmodDiff + Assets map[string][]byte // zip-internal path -> raw content, manifest/readme/image excluded +} + +// ParseExmodz unpacks zipData (an in-memory .EXMODZ) into its manifest and +// bundled assets. The manifest lives at "Extracted Mods/.EXMOD" in +// every sample seen so far; this looks for any "*.EXMOD" file under an +// "Extracted Mods/" prefix rather than hard-coding the mod name, since that +// varies per mod. +func ParseExmodz(zipData []byte) (*ExmodzBundle, error) { + zr, err := zip.NewReader(bytes.NewReader(zipData), int64(len(zipData))) + if err != nil { + return nil, fmt.Errorf("icarus: opening .EXMODZ: %w", err) + } + + bundle := &ExmodzBundle{Assets: make(map[string][]byte)} + var manifestPath string + for _, f := range zr.File { + if strings.HasPrefix(f.Name, "Extracted Mods/") && strings.HasSuffix(f.Name, ".EXMOD") { + manifestPath = f.Name + data, err := readZipFile(f) + if err != nil { + return nil, fmt.Errorf("icarus: reading %s: %w", f.Name, err) + } + bundle.Diff, err = ParseExmod(data) + if err != nil { + return nil, err + } + continue + } + } + if manifestPath == "" { + return nil, fmt.Errorf("icarus: .EXMODZ has no Extracted Mods/*.EXMOD manifest") + } + + for _, f := range zr.File { + if f.Name == manifestPath || f.FileInfo().IsDir() { + continue + } + if !strings.HasSuffix(f.Name, ".uasset") && !strings.HasSuffix(f.Name, ".uexp") { + continue // skip readme/image/other non-asset files — never placed into the output pak + } + data, err := readZipFile(f) + if err != nil { + return nil, fmt.Errorf("icarus: reading asset %s: %w", f.Name, err) + } + bundle.Assets[f.Name] = data + } + + return bundle, nil +} + +func readZipFile(f *zip.File) ([]byte, error) { + rc, err := f.Open() + if err != nil { + return nil, err + } + defer rc.Close() //nolint:errcheck + return io.ReadAll(rc) +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +```bash +go test ./internal/source/icarus/... -v +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add internal/source/icarus/exmodz.go internal/source/icarus/exmodz_test.go +git commit -m "feat: add .EXMODZ archive unpacking (#136)" +``` + +--- + +## Task 12a: Icarus source — base-table dump fetcher + build detection (do BEFORE Task 12) + +Added in rev3. Numbered `12a` so existing task numbers keep their meaning; it is a +prerequisite of Task 12, not a follow-up. + +This is the component that resolves the Oodle blocker: it fetches the base data tables from +the community's per-week dump, works out which build is installed, and refuses to hand +Task 12 a dump that does not match that build. All of it is grounded in spike 3 — see +`docs/plans/icarus-pak-format-findings.md` Part 3 for the fetched URLs and measurements. + +**Files:** + +- Create: `internal/source/icarus/datadump.go` +- Create: `internal/source/icarus/datadump_test.go` + +**Interfaces:** + +- Consumes: `unrealpak.Open`/`Reader.Files`/`Reader.ReadFile` (Tasks 2–3) for validation; + the repo's shared `httpclient` conventions, same as Task 7's Firestore client. +- Produces: `type Build`, `type Dump`, `type DumpStore`, + `func newDumpStore(cacheDir string, httpClient *http.Client) *DumpStore`, + `func (s *DumpStore) DumpForBuild(ctx context.Context, basePakPath, localDumpDir string) (*Dump, error)`, + `func detectBuild(installRoot string) (Build, error)` — Task 12 depends on these names. + +**Design notes (why it looks like this):** + +- **Week resolution is content-based, not name-based.** Nothing in the install records a + week number: `Icarus/Config/version.json` gives `3.0.21.155335`, and Steam's appmanifest + gives a `buildid`, but neither says "Week 243". Steam's news feed does map versions to + weeks, but only in prose titles that will drift. So the store fetches a candidate dump and + _proves_ it matches by byte-comparing the tables `data.pak` stores uncompressed — the 40 + entries readable without Oodle. Exact, offline, and no prose parsing. +- **One dump-tree download, not 298 file fetches.** The tarball is 36 MB and lands in ~4 s; + per-file fetching would be hundreds of round trips. +- **LF → CRLF on ingest.** Dump blobs are LF (committed with autocrlf); shipped paks are + CRLF. Restoring CRLF reproduces shipped bytes exactly. Doing it once at ingest keeps the + rest of the pipeline free of encoding special cases. A local dump directory may already + hold CRLF (QuickBMS writes what the pak stored), so the conversion is idempotent — + CRLF is normalized to LF first, then back — and both sources land in the same shape. +- **Hosted dump primary, local directory override (rev4 — USER DECISION).** A user may point + the pipeline at a directory holding their own unpacked `data.pak` JSON tree (QuickBMS + output, IMM's extracted `data` folder, anything with the same layout). When set, that + directory is used **instead of** the network fetch — same tables, different transport. + This exists because the hosted dump can lag the installed game (it was 7 weeks behind at + spike time), and a user who can unpack their own pak should not be blocked on a third + party. It also makes offline compiles possible. +- **Validation is identical for both sources.** `validateDump` runs on whatever was loaded, + hosted or local. A local directory from the wrong week fails exactly as loudly as a stale + hosted dump, naming the disagreeing tables. The override changes _where tables come from_, + never _whether they are checked_ — silently trusting a user-supplied directory is the + precise failure this gate exists to prevent. + +- [ ] **Step 1: Write the failing tests** + +Serve a synthetic dump tarball and a synthetic pak from `httptest`/`t.TempDir()` so the +tests never touch the network or a real install. + +```go +package icarus + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +// tarGz builds a dump-shaped tarball: a single top-level directory, then the +// table tree beneath it, LF-terminated exactly as the real repo stores it. +func tarGz(t *testing.T, root string, files map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + for name, body := range files { + hdr := &tar.Header{Name: root + "/" + name, Mode: 0o644, Size: int64(len(body))} + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(body)); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func TestDetectBuild_ReadsVersionJSON(t *testing.T) { + root := t.TempDir() + cfg := filepath.Join(root, "Icarus", "Config") + if err := os.MkdirAll(cfg, 0o755); err != nil { + t.Fatal(err) + } + const vjson = `{"Name":"Icarus","Version":{"Major":3,"Minor":0,"Patch":21,` + + `"Changelist":155335,"BuildType":"Shipping","FeatureLevel":"DangerousHorizons"},` + + `"Data":{"Changelist":155151}}` + if err := os.WriteFile(filepath.Join(cfg, "version.json"), []byte(vjson), 0o644); err != nil { + t.Fatal(err) + } + + b, err := detectBuild(root) + if err != nil { + t.Fatalf("detectBuild: %v", err) + } + if got := b.String(); got != "3.0.21.155335" { + t.Errorf("Build.String() = %q, want 3.0.21.155335", got) + } + if b.DataChangelist != 155151 { + t.Errorf("DataChangelist = %d, want 155151", b.DataChangelist) + } +} + +func TestDetectBuild_MissingVersionFile_Errors(t *testing.T) { + if _, err := detectBuild(t.TempDir()); err == nil { + t.Fatal("expected error when version.json is absent, got nil") + } +} + +// A dump whose stored tables match the local pak byte-for-byte (after CRLF +// restoration) is accepted, and its tables are exposed with shipped bytes. +func TestDumpStore_DumpForBuild_AcceptsMatchingDump(t *testing.T) { + const rel = "Factions/D_Factions.json" + shipped := []byte("{\r\n \"Rows\": []\r\n}") // CRLF, as the pak stores it + dumped := "{\n \"Rows\": []\n}" // LF, as the repo stores it + + pak := writeTestBasePak(t, map[string][]byte{rel: shipped}) // Task 12's helper + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(tarGz(t, "IcarusData-abc123", map[string]string{rel: dumped})) + })) + defer srv.Close() + + store := newDumpStore(t.TempDir(), srv.Client()) + store.treeURL = srv.URL // test seam + + dump, err := store.DumpForBuild(context.Background(), pak, "") + if err != nil { + t.Fatalf("DumpForBuild: %v", err) + } + got, ok := dump.Table(rel) + if !ok { + t.Fatalf("dump has no table %q", rel) + } + if !bytes.Equal(got, shipped) { + t.Errorf("table bytes = %q, want the shipped CRLF form %q", got, shipped) + } +} + +// The case that is live today: the newest dump is an older week than the +// install. Must fail loudly and name what disagreed. +func TestDumpStore_DumpForBuild_RejectsWrongWeek(t *testing.T) { + const rel = "Factions/D_Factions.json" + pak := writeTestBasePak(t, map[string][]byte{rel: []byte("{\r\n \"Rows\": [1]\r\n}")}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(tarGz(t, "IcarusData-old", map[string]string{rel: "{\n \"Rows\": []\n}"})) + })) + defer srv.Close() + + store := newDumpStore(t.TempDir(), srv.Client()) + store.treeURL = srv.URL + + _, err := store.DumpForBuild(context.Background(), pak, "") + if err == nil { + t.Fatal("expected an error for a dump that does not match the install, got nil") + } + if !strings.Contains(err.Error(), rel) { + t.Errorf("error %q should name the table that disagreed (%s)", err, rel) + } +} + +// writeLocalDump lays out an unpacked-data.pak-shaped directory on disk. +func writeLocalDump(t *testing.T, files map[string]string) string { + t.Helper() + dir := t.TempDir() + for rel, body := range files { + full := filepath.Join(dir, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + return dir +} + +// With a local dump directory configured, the network is never touched. +func TestDumpStore_DumpForBuild_LocalDirOverridesFetch(t *testing.T) { + const rel = "Factions/D_Factions.json" + shipped := []byte("{\r\n \"Rows\": []\r\n}") + pak := writeTestBasePak(t, map[string][]byte{rel: shipped}) + local := writeLocalDump(t, map[string]string{rel: "{\n \"Rows\": []\n}"}) + + fetched := false + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fetched = true + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + store := newDumpStore(t.TempDir(), srv.Client()) + store.treeURL = srv.URL + + dump, err := store.DumpForBuild(context.Background(), pak, local) + if err != nil { + t.Fatalf("DumpForBuild with a local dump dir: %v", err) + } + if fetched { + t.Error("the hosted dump was fetched even though a local dump dir was configured") + } + got, ok := dump.Table(rel) + if !ok || !bytes.Equal(got, shipped) { + t.Errorf("table bytes = %q (found=%v), want the shipped CRLF form %q", got, ok, shipped) + } +} + +// A local directory already storing CRLF must load unchanged — QuickBMS writes +// whatever the pak stored, so the conversion has to be idempotent. +func TestDumpStore_DumpForBuild_LocalDirAlreadyCRLF(t *testing.T) { + const rel = "Factions/D_Factions.json" + shipped := "{\r\n \"Rows\": []\r\n}" + pak := writeTestBasePak(t, map[string][]byte{rel: []byte(shipped)}) + local := writeLocalDump(t, map[string]string{rel: shipped}) + + store := newDumpStore(t.TempDir(), http.DefaultClient) + store.treeURL = "http://127.0.0.1:0/never-used" + + if _, err := store.DumpForBuild(context.Background(), pak, local); err != nil { + t.Fatalf("DumpForBuild with a CRLF local dump dir: %v", err) + } +} + +// A local dir from the wrong week is rejected exactly like a stale hosted +// dump, and the error points at the configured path. +func TestDumpStore_DumpForBuild_LocalDirWrongWeek_Rejected(t *testing.T) { + const rel = "Factions/D_Factions.json" + pak := writeTestBasePak(t, map[string][]byte{rel: []byte("{\r\n \"Rows\": [1]\r\n}")}) + local := writeLocalDump(t, map[string]string{rel: "{\n \"Rows\": []\n}"}) + + store := newDumpStore(t.TempDir(), http.DefaultClient) + store.treeURL = "http://127.0.0.1:0/never-used" + + _, err := store.DumpForBuild(context.Background(), pak, local) + if err == nil { + t.Fatal("expected an error for a local dump dir from a different week, got nil") + } + if !strings.Contains(err.Error(), rel) { + t.Errorf("error %q should name the disagreeing table (%s)", err, rel) + } + if !strings.Contains(err.Error(), local) { + t.Errorf("error %q should name the configured data_dump_path (%s)", err, local) + } +} + +func TestDumpStore_DumpForBuild_LocalDirEmpty_IsActionable(t *testing.T) { + pak := writeTestBasePak(t, map[string][]byte{"a/B.json": []byte("{}")}) + store := newDumpStore(t.TempDir(), http.DefaultClient) + + _, err := store.DumpForBuild(context.Background(), pak, t.TempDir()) + if err == nil { + t.Fatal("expected an error for a data_dump_path holding no JSON tables, got nil") + } +} + +func TestDumpStore_DumpForBuild_NetworkFailure_IsActionable(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + store := newDumpStore(t.TempDir(), srv.Client()) + store.treeURL = srv.URL + + pak := writeTestBasePak(t, map[string][]byte{"a/B.json": []byte("{}")}) + _, err := store.DumpForBuild(context.Background(), pak, "") + if err == nil { + t.Fatal("expected an error when the dump host fails, got nil") + } +} +``` + +- [ ] **Step 2: Run to verify they fail** + +```bash +go test ./internal/source/icarus/... -run 'TestDetectBuild|TestDumpStore' -v +``` + +Expected: FAIL (`detectBuild`, `newDumpStore` undefined). + +- [ ] **Step 3: Implement `datadump.go`** + +```go +package icarus + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "io/fs" + "net/http" + "os" + "path" + "path/filepath" + "sort" + "strings" + + "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" +) + +// defaultDumpTreeURL is the community per-week unpack of Icarus's data.pak: +// https://github.com/GODOFMINECRAFT4/IcarusData. The tree is committed as +// loose JSON at the repo root, one commit per game week, with the week +// recorded only in the commit message — there are no tags or releases. This +// URL is HEAD; a specific week is addressed by substituting its commit SHA. +const defaultDumpTreeURL = "https://codeload.github.com/GODOFMINECRAFT4/IcarusData/tar.gz/refs/heads/master" + +// maxDumpBytes caps the download. The real tree is ~36 MB; this leaves room to +// grow while refusing to stream an unbounded body into memory. +const maxDumpBytes = 256 << 20 + +// Build identifies the installed game, read from Icarus/Config/version.json. +// Note this carries no week number — nothing in the install does. Week +// agreement is established by content comparison, not by this value. +type Build struct { + Major, Minor, Patch int + Changelist int + DataChangelist int + FeatureLevel string +} + +func (b Build) String() string { + return fmt.Sprintf("%d.%d.%d.%d", b.Major, b.Minor, b.Patch, b.Changelist) +} + +// detectBuild reads /Icarus/Config/version.json. +func detectBuild(installRoot string) (Build, error) { + p := filepath.Join(installRoot, "Icarus", "Config", "version.json") + raw, err := os.ReadFile(p) + if err != nil { + return Build{}, fmt.Errorf("icarus: reading game version from %s: %w", p, err) + } + var doc struct { + Version struct { + Major, Minor, Patch int + Changelist int + FeatureLevel string + } + Data struct{ Changelist int } + } + if err := json.Unmarshal(raw, &doc); err != nil { + return Build{}, fmt.Errorf("icarus: parsing %s: %w", p, err) + } + return Build{ + Major: doc.Version.Major, Minor: doc.Version.Minor, Patch: doc.Version.Patch, + Changelist: doc.Version.Changelist, + DataChangelist: doc.Data.Changelist, + FeatureLevel: doc.Version.FeatureLevel, + }, nil +} + +// Dump is a fetched set of base data tables, keyed by mount-relative path +// (e.g. "Factions/D_Factions.json") with values already converted back to the +// game's CRLF line endings. +type Dump struct { + tables map[string][]byte +} + +// Table returns one table's shipped bytes. +func (d *Dump) Table(rel string) ([]byte, bool) { + b, ok := d.tables[rel] + return b, ok +} + +// DumpStore fetches and caches base-table dumps. +type DumpStore struct { + cacheDir string + httpClient *http.Client + treeURL string // overridable in tests +} + +func newDumpStore(cacheDir string, httpClient *http.Client) *DumpStore { + return &DumpStore{cacheDir: cacheDir, httpClient: httpClient, treeURL: defaultDumpTreeURL} +} + +// DumpForBuild loads the base data tables and returns them only if they match +// the installed game, proven by byte-comparing every table basePakPath stores +// uncompressed. A mismatch means the tables are for a different game week: +// that is a hard error naming the offending tables, never a silent +// best-effort. +// +// localDumpDir, when non-empty, is a user-supplied directory holding an +// unpacked data.pak JSON tree (QuickBMS output and the like); it replaces the +// network fetch entirely. Validation is the same either way — a local +// directory from the wrong week is rejected exactly like a stale hosted dump. +func (s *DumpStore) DumpForBuild(ctx context.Context, basePakPath, localDumpDir string) (*Dump, error) { + var ( + dump *Dump + err error + ) + if localDumpDir != "" { + dump, err = loadLocalDump(localDumpDir) + } else { + dump, err = s.fetchTree(ctx, s.treeURL) + } + if err != nil { + return nil, err + } + if err := validateDump(dump, basePakPath); err != nil { + if localDumpDir != "" { + return nil, fmt.Errorf("%w (tables were read from the configured data_dump_path %s)", err, localDumpDir) + } + return nil, err + } + return dump, nil +} + +// loadLocalDump reads an unpacked data.pak JSON tree from disk. The layout is +// the same one the hosted dump ships — table paths relative to the directory +// root, e.g. "Factions/D_Factions.json" — so a user can point this at QuickBMS +// output without rearranging anything. +func loadLocalDump(dir string) (*Dump, error) { + info, err := os.Stat(dir) + if err != nil { + return nil, fmt.Errorf("icarus: reading the configured data_dump_path %s: %w", dir, err) + } + if !info.IsDir() { + return nil, fmt.Errorf("icarus: the configured data_dump_path %s is not a directory", dir) + } + + dump := &Dump{tables: make(map[string][]byte)} + err = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(d.Name(), ".json") { + return nil + } + rel, err := filepath.Rel(dir, p) + if err != nil { + return err + } + body, err := os.ReadFile(p) + if err != nil { + return err + } + dump.tables[filepath.ToSlash(rel)] = toCRLF(body) + return nil + }) + if err != nil { + return nil, fmt.Errorf("icarus: scanning the configured data_dump_path %s: %w", dir, err) + } + if len(dump.tables) == 0 { + return nil, fmt.Errorf("icarus: the configured data_dump_path %s contains no JSON tables "+ + "(expected an unpacked data.pak tree, e.g. Factions/D_Factions.json)", dir) + } + return dump, nil +} + +// fetchTree downloads a dump tarball and ingests its JSON tables, restoring +// the CRLF line endings the game ships (the repo stores LF). +func (s *DumpStore) fetchTree(ctx context.Context, url string) (*Dump, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("icarus: building dump request: %w", err) + } + resp, err := s.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("icarus: fetching base-table dump: %w "+ + "(compiling Icarus mods requires network access — see the plan's Global Constraints)", err) + } + defer resp.Body.Close() //nolint:errcheck + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("icarus: fetching base-table dump from %s: HTTP %d", url, resp.StatusCode) + } + + zr, err := gzip.NewReader(io.LimitReader(resp.Body, maxDumpBytes)) + if err != nil { + return nil, fmt.Errorf("icarus: base-table dump is not valid gzip: %w", err) + } + defer zr.Close() //nolint:errcheck + + dump := &Dump{tables: make(map[string][]byte)} + tr := tar.NewReader(zr) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("icarus: reading base-table dump: %w", err) + } + if hdr.Typeflag != tar.TypeReg || !strings.HasSuffix(hdr.Name, ".json") { + continue + } + // Strip the archive's single top-level directory (e.g. + // "IcarusData-/") to get the mount-relative table path. + rel := hdr.Name + if i := strings.Index(rel, "/"); i >= 0 { + rel = rel[i+1:] + } + // The repo also carries a stale "data/" copy of the tree; the + // authoritative tables are the root-level ones. + if rel == "" || strings.HasPrefix(rel, "data/") { + continue + } + body, err := io.ReadAll(tr) + if err != nil { + return nil, fmt.Errorf("icarus: reading %s from base-table dump: %w", rel, err) + } + dump.tables[path.Clean(rel)] = toCRLF(body) + } + if len(dump.tables) == 0 { + return nil, fmt.Errorf("icarus: base-table dump from %s contained no JSON tables", url) + } + return dump, nil +} + +// toCRLF restores the game's line endings. The dump repo stores LF (committed +// with autocrlf); the shipped pak stores CRLF, and the two are otherwise +// byte-identical. Existing CRLFs are left alone so the conversion is +// idempotent. +func toCRLF(b []byte) []byte { + return []byte(strings.ReplaceAll(strings.ReplaceAll(string(b), "\r\n", "\n"), "\n", "\r\n")) +} + +// validateDump proves a dump belongs to the installed game. +// +// Only the tables data.pak stores *uncompressed* can be checked — the rest are +// Oodle-compressed and unreadable here, which is the whole reason the dump +// exists. That is enough: a dump built from a different week's data.pak +// disagrees on some of them, and in practice it disagrees loudly (the spike saw +// 3 differing stored tables and 6 missing tables across a 7-week gap). +func validateDump(dump *Dump, basePakPath string) error { + pak, err := unrealpak.Open(basePakPath) + if err != nil { + return fmt.Errorf("icarus: opening base pak %s for dump validation: %w", basePakPath, err) + } + defer pak.Close() //nolint:errcheck + + var missing, differing []string + checked := 0 + for _, f := range pak.Files() { + shipped, err := pak.ReadFile(f.Path) + if err != nil { + if errors.Is(err, unrealpak.ErrUnsupportedFormat) { + continue // Oodle-compressed (or similar): not readable here, and not our gate + } + // Any other ReadFile failure — corruption, a truncated payload, an + // I/O error — is not an expected skip. Silently excluding it here + // would quietly narrow what this gate actually verified, exactly + // the "no silent fallbacks" failure this function exists to prevent. + return fmt.Errorf("icarus: validating base pak %s: reading %s: %w", basePakPath, f.Path, err) + } + checked++ + got, ok := dump.Table(f.Path) + if !ok { + missing = append(missing, f.Path) + continue + } + if !bytes.Equal(got, shipped) { + differing = append(differing, f.Path) + } + } + if checked == 0 { + return fmt.Errorf("icarus: %s exposed no uncompressed tables to validate the dump against", basePakPath) + } + if len(missing) == 0 && len(differing) == 0 { + return nil + } + sort.Strings(missing) + sort.Strings(differing) + return fmt.Errorf( + "icarus: the available base-table dump does not match the installed game "+ + "(%d/%d uncompressed tables disagree: %s). The dump is for a different game week. "+ + "Wait for the dump to be updated for your game version, or roll the game back to a "+ + "matching week; compiling against a mismatched week would silently corrupt mod data", + len(missing)+len(differing), checked, summarize(append(differing, missing...))) +} + +func summarize(paths []string) string { + const max = 3 + if len(paths) <= max { + return strings.Join(paths, ", ") + } + return fmt.Sprintf("%s and %d more", strings.Join(paths[:max], ", "), len(paths)-max) +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +```bash +go test ./internal/source/icarus/... -run 'TestDetectBuild|TestDumpStore' -v +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add internal/source/icarus/datadump.go internal/source/icarus/datadump_test.go +git commit -m "feat: fetch Icarus base data tables from the per-week community dump (#136)" +``` + +--- + +## Task 12: Compile orchestration + +**Files:** + +- Create: `internal/source/icarus/compile.go` +- Create: `internal/source/icarus/compile_test.go` + +**Interfaces:** + +- Consumes: `unrealpak.Open`, `Reader.Files`, `unrealpak.Create`, `Writer.AddFile`/`Close` (Tasks 2–4); `ParseExmodz` (Task 11); `ApplyRowPatch` (Task 10); `DumpStore.DumpForBuild` (Task 12a). +- Produces: `func Compile(ctx context.Context, dumps *DumpStore, basePakPath, localDumpDir, exmodzPath, outputPakPath string) error` — Task 13 depends on this exact signature. This is also the function `source.Compiler` (Task 13) wraps; `localDumpDir` is the game's optional `data_dump_path` and is `""` when unset. (Implemented with a named return, `func Compile(...) (err error)`, used internally by a deferred cleanup on failure — see "Partial-output cleanup on failure" below; the call-site type is unchanged, so this is invisible to Task 13.) + +> **Base tables come from a hosted per-week dump (rev3 — USER DECISION; resolves the Oodle +> blocker).** The base pak is `Icarus/Content/Data/data.pak` (298 files, all `.json`), and +> **258 of them are Oodle-compressed** — including every table a mod would plausibly patch +> (`D_ItemsStatic` 7.3 MB, `D_Talents` 2.6 MB, `D_Quests` 1.2 MB). Only 40 tiny stubs are +> stored uncompressed. Oodle has no Go stdlib decoder, so the base tables **cannot** be read +> out of the local pak. +> +> Resolution: fetch the base tables from the community's per-week JSON dump instead +> (Task 12a). The local `data.pak` is still opened, but for **validation only** — its 40 +> stored tables are readable without Oodle and are byte-compared against the dump to prove +> the dump is the right week. That check is not speculative complexity: it is exactly how +> the spike detected that the newest dump is 7 weeks behind a current install. +> +> Two facts from the spike that shape this task's error handling — see +> `docs/plans/icarus-pak-format-findings.md` Part 3: +> +> - Dump blobs are **LF**; the shipped pak is **CRLF**. The fetcher restores `LF -> CRLF`, +> which reproduces shipped bytes exactly (37/40 stored tables byte-identical). +> - **A matching dump may simply not exist yet.** At spike time the install was Week 243 and +> the freshest dump was Week 236. "No dump for the installed build" is a normal outcome +> that must fail loudly with an actionable message — not an edge case, and never a silent +> fall back to a mismatched week. + +**Note on the base data-table's mount path (corrected against real data — see task-12-report.md "plan delta 1")**: the real sample's `Rows[].CurrentFile` is the mount-relative directory path with every `/` flattened to `-` (e.g. base pak path `AI/D_AIGrowth.json` is recorded as `AI-D_AIGrowth.json`; a deeper path like `Audio/MusicConditions/D_MusicLocationConditions.json` is recorded as `Audio-MusicConditions-D_MusicLocationConditions.json`) — not a suffix-matchable bare filename as originally assumed here. A literal `/` suffix match, verified against a real install and a real `.EXMODZ`, matches **0 of 14** real rows. `resolveCurrentFile`/`matchMountPath` instead reconstruct the mount path by reversing that substitution (`strings.ReplaceAll(currentFile, "-", "/")`) and doing an **exact** match against the base pak's file listing; this is unambiguous in practice (verified: none of Icarus's 298 real base-table paths contain a literal hyphen). Zero matches or more than one match is still a loud, named error — the ambiguous-match case is now structurally unreachable through `unrealpak.Writer`'s public API (it rejects duplicate mount paths), so it is exercised as a `matchMountPath`-level unit test with a hand-built duplicate-path slice rather than a full `Compile()` integration test. + +**`EndOfMod` sentinel**: real `.EXMOD` manifests terminate their `Rows` array with `{"CurrentFile":"EndOfMod"}` and no `File_Items` key at all — a known ecosystem terminator, not a data-table row. `Compile`'s row loop skips it explicitly (`row.CurrentFile == "EndOfMod"`, checked before `resolveCurrentFile` is ever called); any _other_ row with zero `File_Items` is treated as a malformed manifest and fails loudly, naming the row. + +**Asset-path sanitation (controller-flagged security item, resolved — see task-12-report.md "plan delta 2")**: `ParseExmodz` (Task 11) carries each bundled asset's raw zip entry name through unchanged as the `ExmodzBundle.Assets` map key, and neither it nor `unrealpak.Writer.AddFile` sanitizes it. Before writing any asset, `Compile` calls `sanitizeAssetPath`, which normalizes backslashes to `/`, rejects a NUL byte, rejects absolute paths (a leading `/` or a Windows drive form like `C:/...`), and `path.Clean`s the result, rejecting anything that is `.`, `..`, or escapes with a leading `../`. This closes a pak-slip vector where a crafted `.EXMODZ` asset entry (`../evil.uasset`, `/evil`, `C:\evil`) could escape the mod's own namespace once the pak is deployed or unpacked elsewhere. + +**Partial-output cleanup on failure (fix round 1 — see task-12-report.md)**: `unrealpak.Create` opens `outputPakPath` eagerly, so any error after that point (in either loop, or in `out.Close()`) originally left a partial/incomplete pak on disk — a hazard, since it could be picked up and deployed. `Compile` uses a named return, `func Compile(...) (err error)`, with a `defer` that removes `outputPakPath` on any non-nil error, joining a removal failure into the returned error rather than masking it; the success path is untouched. `unrealpak.Writer` has no way to abort without finalizing (`Close` always serializes and writes whatever was buffered, producing a semantically-incomplete-but-valid pak — worse than an empty file), so `os.Remove` on the error path is the fix, not a `unrealpak` API change; the underlying file descriptor is only reclaimed on GC in this path, a known, accepted limitation. + +- [ ] **Step 1: Write the failing test** + +```go +package icarus + +import ( + "archive/zip" + "bytes" + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" +) + +// testDumpStore serves a dump containing exactly files, so validateDump agrees +// it matches the base pak built from the same map. Reuses tarGz from +// datadump_test.go (same package). +func testDumpStore(t *testing.T, files map[string][]byte) *DumpStore { + t.Helper() + entries := make(map[string]string, len(files)) + for name, data := range files { + entries[name] = string(data) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(tarGz(t, "IcarusData-test", entries)) + })) + t.Cleanup(srv.Close) + store := newDumpStore(t.TempDir(), srv.Client()) + store.treeURL = srv.URL + return store +} + +// writeTestBasePak is defined in datadump_test.go (same package) — Task 12a +// needed it first for its own tests, so this file reuses it rather than +// redeclaring it. + +func writeTestExmodzFile(t *testing.T, manifestJSON string, assets map[string][]byte) string { + t.Helper() + path := filepath.Join(t.TempDir(), "mod.exmodz") + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, _ := zw.Create("Extracted Mods/Test.EXMOD") + w.Write([]byte(manifestJSON)) //nolint:errcheck + for name, data := range assets { + aw, _ := zw.Create(name) + aw.Write(data) //nolint:errcheck + } + zw.Close() //nolint:errcheck + if err := os.WriteFile(path, buf.Bytes(), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +// Base table paths and CurrentFile values below mirror the real shape found +// against a live install + a real Bear_Mount.EXMODZ during Step 5b +// verification: CurrentFile flattens the mount-relative directory path with +// "-" in place of "/" (e.g. "AI-D_AIGrowth.json" for base pak path +// "AI/D_AIGrowth.json"), not a bare filename living at a hyphenated leaf as +// originally assumed. See task-12-report.md "plan delta". +func TestCompile_AppliesDiffAndBundlesAssets(t *testing.T) { + baseTables := map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":200}}`), + } + basePak := writeTestBasePak(t, baseTables) + dumps := testDumpStore(t, baseTables) + manifest := `{"name":"Bear Mount","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":235}]}]}` + exmodzPath := writeTestExmodzFile(t, manifest, map[string][]byte{ + "Bear_Mount/ASS/ITM/SK_ITM_Saddle_Bear.uasset": []byte("fake-asset"), + }) + outputPath := filepath.Join(t.TempDir(), "Bear_Mount_P.pak") + + if err := Compile(context.Background(), dumps, basePak, "", exmodzPath, outputPath); err != nil { + t.Fatalf("Compile: %v", err) + } + + r, err := unrealpak.Open(outputPath) + if err != nil { + t.Fatalf("opening compiled output: %v", err) + } + defer r.Close() + + patched, err := r.ReadFile("AI/D_AIGrowth.json") + if err != nil { + t.Fatalf("ReadFile patched data table: %v", err) + } + if !bytes.Contains(patched, []byte(`"BaseMovementSpeed":235`)) { + t.Errorf("patched data table = %s, want BaseMovementSpeed 235", patched) + } + + asset, err := r.ReadFile("Bear_Mount/ASS/ITM/SK_ITM_Saddle_Bear.uasset") + if err != nil { + t.Fatalf("ReadFile bundled asset: %v", err) + } + if string(asset) != "fake-asset" { + t.Errorf("bundled asset content = %q", asset) + } +} + +// The real .EXMOD ecosystem terminates Rows with {"CurrentFile":"EndOfMod"} +// and no File_Items key — Compile must skip it, not try to resolve it as a +// data table (it has none). +func TestCompile_SkipsEndOfModSentinelRow(t *testing.T) { + baseTables := map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":200}}`), + } + basePak := writeTestBasePak(t, baseTables) + dumps := testDumpStore(t, baseTables) + manifest := `{"name":"X","Rows":[` + + `{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":235}]},` + + `{"CurrentFile":"EndOfMod"}]}` + exmodzPath := writeTestExmodzFile(t, manifest, nil) + outputPath := filepath.Join(t.TempDir(), "out.pak") + + if err := Compile(context.Background(), dumps, basePak, "", exmodzPath, outputPath); err != nil { + t.Fatalf("Compile: %v", err) + } + + r, err := unrealpak.Open(outputPath) + if err != nil { + t.Fatalf("opening compiled output: %v", err) + } + defer r.Close() + patched, err := r.ReadFile("AI/D_AIGrowth.json") + if err != nil { + t.Fatalf("ReadFile patched data table: %v", err) + } + if !bytes.Contains(patched, []byte(`"BaseMovementSpeed":235`)) { + t.Errorf("patched data table = %s, want BaseMovementSpeed 235", patched) + } +} + +// A real (non-sentinel) row with no File_Items is a malformed manifest, not +// something to silently skip — only the EndOfMod sentinel gets that pass. +func TestCompile_RowWithoutFileItems_Errors(t *testing.T) { + baseTables := map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":200}}`), + } + basePak := writeTestBasePak(t, baseTables) + dumps := testDumpStore(t, baseTables) + manifest := `{"name":"X","Rows":[{"CurrentFile":"AI-D_AIGrowth.json"}]}` + exmodzPath := writeTestExmodzFile(t, manifest, nil) + outputPath := filepath.Join(t.TempDir(), "out.pak") + + err := Compile(context.Background(), dumps, basePak, "", exmodzPath, outputPath) + if err == nil { + t.Fatal("expected an error for a non-sentinel row with no File_Items, got nil") + } + if !strings.Contains(err.Error(), "AI-D_AIGrowth.json") { + t.Errorf("error %q should name the offending row", err) + } +} + +// A stale dump must stop the compile before any output pak is written — this +// is the live case today, where the newest dump lags the installed game. +func TestCompile_DumpWeekMismatch_FailsBeforeWriting(t *testing.T) { + basePak := writeTestBasePak(t, map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":200}}`), + }) + dumps := testDumpStore(t, map[string][]byte{ // different week's content + "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":150}}`), + }) + manifest := `{"name":"X","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":235}]}]}` + exmodzPath := writeTestExmodzFile(t, manifest, nil) + outputPath := filepath.Join(t.TempDir(), "out.pak") + + err := Compile(context.Background(), dumps, basePak, "", exmodzPath, outputPath) + if err == nil { + t.Fatal("expected an error when the dump is for a different game week, got nil") + } + if _, statErr := os.Stat(outputPath); statErr == nil { + t.Error("no output pak should exist after a week-mismatch failure") + } +} + +// A malicious .EXMODZ whose bundled asset entry escapes the mod's own path +// must fail loudly rather than write outside the pak's intended namespace — +// see task-12-report.md "plan delta" for the exact semantics. +func TestCompile_UnsafeAssetPath_Errors(t *testing.T) { + baseTables := map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":200}}`), + } + basePak := writeTestBasePak(t, baseTables) + dumps := testDumpStore(t, baseTables) + manifest := `{"name":"X","Rows":[]}` + exmodzPath := writeTestExmodzFile(t, manifest, map[string][]byte{ + "../evil.uasset": []byte("payload"), + }) + outputPath := filepath.Join(t.TempDir(), "out.pak") + + err := Compile(context.Background(), dumps, basePak, "", exmodzPath, outputPath) + if err == nil { + t.Fatal("expected an error for an asset path escaping the mod's own namespace, got nil") + } + if !strings.Contains(err.Error(), "../evil.uasset") { + t.Errorf("error %q should name the offending asset path", err) + } + if _, statErr := os.Stat(outputPath); statErr == nil { + t.Error("no partial output pak should exist after an unsafe-asset-path failure") + } +} + +// A failure that happens after unrealpak.Create(outputPakPath) has already +// created the file on disk (here: an unresolvable row, mid row-loop) must +// not leave a partial/incomplete pak behind — a stray partial _P.pak is a +// hazard (it could be picked up and deployed) and contradicts the +// fail-loud-and-clean philosophy. See task-12-report.md "plan delta" (fix +// round 1). +func TestCompile_MidCompileFailure_LeavesNoOutputFile(t *testing.T) { + baseTables := map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":200}}`), + } + basePak := writeTestBasePak(t, baseTables) + dumps := testDumpStore(t, baseTables) + // CurrentFile has no matching base-pak file: resolveCurrentFile fails + // inside the row loop, after out has already been created. + manifest := `{"name":"X","Rows":[{"CurrentFile":"AI-D_Nonexistent.json","File_Items":[{"Name":"Mount_Bear","X":1}]}]}` + exmodzPath := writeTestExmodzFile(t, manifest, nil) + outputPath := filepath.Join(t.TempDir(), "out.pak") + + err := Compile(context.Background(), dumps, basePak, "", exmodzPath, outputPath) + if err == nil { + t.Fatal("expected an error for an unresolvable row, got nil") + } + if _, statErr := os.Stat(outputPath); statErr == nil { + t.Error("no partial output pak should exist after a mid-compile failure") + } else if !os.IsNotExist(statErr) { + t.Errorf("unexpected error stat-ing output path: %v", statErr) + } +} + +func TestMatchMountPath(t *testing.T) { + paths := []string{ + "AI/D_AIGrowth.json", + "Audio/MusicConditions/D_MusicLocationConditions.json", + "D_Factions.json", + } + + t.Run("single-level directory", func(t *testing.T) { + got, err := matchMountPath(paths, "AI-D_AIGrowth.json") + if err != nil { + t.Fatalf("matchMountPath: %v", err) + } + if got != "AI/D_AIGrowth.json" { + t.Errorf("matchMountPath = %q, want AI/D_AIGrowth.json", got) + } + }) + + t.Run("multi-level directory", func(t *testing.T) { + got, err := matchMountPath(paths, "Audio-MusicConditions-D_MusicLocationConditions.json") + if err != nil { + t.Fatalf("matchMountPath: %v", err) + } + if got != "Audio/MusicConditions/D_MusicLocationConditions.json" { + t.Errorf("matchMountPath = %q, want Audio/MusicConditions/D_MusicLocationConditions.json", got) + } + }) + + t.Run("root-level file, no hyphen to convert", func(t *testing.T) { + got, err := matchMountPath(paths, "D_Factions.json") + if err != nil { + t.Fatalf("matchMountPath: %v", err) + } + if got != "D_Factions.json" { + t.Errorf("matchMountPath = %q, want D_Factions.json", got) + } + }) + + t.Run("no match is a loud, actionable error", func(t *testing.T) { + _, err := matchMountPath(paths, "AI-D_Nonexistent.json") + if err == nil { + t.Fatal("expected an error for a CurrentFile with no matching base pak file, got nil") + } + if !strings.Contains(err.Error(), "AI-D_Nonexistent.json") || !strings.Contains(err.Error(), "AI/D_Nonexistent.json") { + t.Errorf("error %q should name both the CurrentFile and the expected mount path", err) + } + }) + + t.Run("ambiguous match is a loud error", func(t *testing.T) { + dup := []string{"AI/D_AIGrowth.json", "AI/D_AIGrowth.json"} + _, err := matchMountPath(dup, "AI-D_AIGrowth.json") + if err == nil { + t.Fatal("expected an error for an ambiguous match, got nil") + } + }) +} + +func TestSanitizeAssetPath(t *testing.T) { + tests := []struct { + name string + raw string + want string + wantErr bool + }{ + {name: "parent traversal", raw: "../evil.json", wantErr: true}, + {name: "absolute unix path", raw: "/evil", wantErr: true}, + {name: "windows drive absolute", raw: `C:\evil`, wantErr: true}, + {name: "backslash-normalized nested path", raw: `Good\Nested\file.uasset`, want: "Good/Nested/file.uasset"}, + {name: "benign nested path", raw: "Bear_Mount/ASS/ITM/SK_ITM_Saddle_Bear.uasset", want: "Bear_Mount/ASS/ITM/SK_ITM_Saddle_Bear.uasset"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := sanitizeAssetPath(tt.raw) + if tt.wantErr { + if err == nil { + t.Fatalf("sanitizeAssetPath(%q) = %q, nil; want error", tt.raw, got) + } + return + } + if err != nil { + t.Fatalf("sanitizeAssetPath(%q): %v", tt.raw, err) + } + if got != tt.want { + t.Errorf("sanitizeAssetPath(%q) = %q, want %q", tt.raw, got, tt.want) + } + }) + } +} +``` + +- [ ] **Step 2: Run to verify it fails** + +```bash +go test ./internal/source/icarus/... -run TestCompile -v +``` + +Expected: FAIL (`Compile` undefined). + +- [ ] **Step 3: Implement** + +```go +package icarus + +import ( + "context" + "fmt" + "os" + "path" + "strings" + + "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" +) + +// Compile reads exmodzPath's .EXMOD diff, applies it to the game's base data +// tables, bundles in any pre-built assets the .EXMODZ carries, and writes the +// result as a new pak at outputPakPath ready to deploy as-is. +// +// The base tables come from the community per-week dump (Task 12a), not from +// basePakPath: 258 of the 298 tables in a real data.pak are Oodle-compressed +// and cannot be read with the stdlib. basePakPath is still opened, for two +// things it alone can answer — which tables the installed game actually has +// (so a bare, hyphen-flattened CurrentFile resolves to a real mount path), +// and whether the dump +// is for the installed week (DumpForBuild byte-checks it against the tables +// the pak stores uncompressed). A dump that does not match fails the whole +// compile; see Task 12a. +// +// localDumpDir is the game's optional data_dump_path: when set, base tables +// are read from that directory instead of being fetched. It is validated +// identically, so a stale local directory fails just as loudly. +func Compile(ctx context.Context, dumps *DumpStore, basePakPath, localDumpDir, exmodzPath, outputPakPath string) (err error) { + exmodzData, err := os.ReadFile(exmodzPath) + if err != nil { + return fmt.Errorf("icarus: reading %s: %w", exmodzPath, err) + } + bundle, err := ParseExmodz(exmodzData) + if err != nil { + return fmt.Errorf("icarus: %s: %w", exmodzPath, err) + } + + base, err := unrealpak.Open(basePakPath) + if err != nil { + return fmt.Errorf("icarus: opening base pak %s: %w", basePakPath, err) + } + defer base.Close() + + // Loaded and validated before anything is written, so a week mismatch or + // an offline machine fails before a half-built pak exists on disk. + dump, err := dumps.DumpForBuild(ctx, basePakPath, localDumpDir) + if err != nil { + return err + } + + out, err := unrealpak.Create(outputPakPath) + if err != nil { + return fmt.Errorf("icarus: creating %s: %w", outputPakPath, err) + } + // unrealpak.Create opens the file eagerly, so any error from here on + // leaves a partial/incomplete pak at outputPakPath unless removed — a + // hazard, since it could be picked up and deployed. unrealpak.Writer has + // no way to abort without finalizing (Close always serializes and writes + // whatever was buffered), so removing the file is the only way to keep + // the fail-loud-and-clean contract on this path; the success path + // (err == nil here) is untouched. + defer func() { + if err == nil { + return + } + if rmErr := os.Remove(outputPakPath); rmErr != nil && !os.IsNotExist(rmErr) { + err = fmt.Errorf("%w (additionally, removing partial output %s failed: %v)", err, outputPakPath, rmErr) + } + }() + + for _, row := range bundle.Diff.Rows { + if row.CurrentFile == endOfModSentinel { + // A known .EXMOD ecosystem terminator row: no File_Items, no + // corresponding data table. Not a row to resolve or patch. + continue + } + if len(row.FileItems) == 0 { + return fmt.Errorf("icarus: %s: row has no File_Items to apply (malformed .EXMOD manifest)", row.CurrentFile) + } + mountPath, err := resolveCurrentFile(base, row.CurrentFile) + if err != nil { + return err + } + baseData, ok := dump.Table(mountPath) + if !ok { + return fmt.Errorf("icarus: base data table %s is present in the installed game "+ + "but missing from the base-table dump", mountPath) + } + patched, err := ApplyRowPatch(baseData, row) + if err != nil { + return err + } + if err := out.AddFile(mountPath, patched); err != nil { + return fmt.Errorf("icarus: writing patched %s: %w", mountPath, err) + } + } + + for assetPath, data := range bundle.Assets { + safePath, err := sanitizeAssetPath(assetPath) + if err != nil { + return err + } + if err := out.AddFile(safePath, data); err != nil { + return fmt.Errorf("icarus: writing bundled asset %s: %w", safePath, err) + } + } + + if err := out.Close(); err != nil { + return fmt.Errorf("icarus: finalizing %s: %w", outputPakPath, err) + } + return nil +} + +// endOfModSentinel is a known .EXMOD ecosystem terminator row: real-world +// manifests end their Rows array with {"CurrentFile":"EndOfMod"} and no +// File_Items key at all. It targets no data table and carries no patch, so +// Compile skips it rather than trying (and failing) to resolve it. +const endOfModSentinel = "EndOfMod" + +// resolveCurrentFile finds the base-pak file a row's bare CurrentFile refers +// to. The .EXMOD schema flattens the mount-relative directory path into +// CurrentFile by replacing every "/" with "-" (e.g. the real base pak path +// "Audio/MusicConditions/D_MusicLocationConditions.json" is recorded as +// "Audio-MusicConditions-D_MusicLocationConditions.json"); reversing that +// substitution reconstructs the mount path exactly. This was verified +// against a real install and a real .EXMODZ: none of Icarus's 298 real base +// table paths contain a literal hyphen, so the reverse mapping is +// unambiguous. Fails loudly on zero or multiple matches — see this task's +// header note; guessing which one is correct is exactly the kind of silent +// fallback repo precedent #95 forbids. +func resolveCurrentFile(base *unrealpak.Reader, currentFile string) (string, error) { + files := base.Files() + paths := make([]string, len(files)) + for i, f := range files { + paths[i] = f.Path + } + return matchMountPath(paths, currentFile) +} + +// matchMountPath resolves currentFile against paths, isolated from +// *unrealpak.Reader so the zero/ambiguous-match error paths can be tested +// directly without needing a base pak with (unreachable in valid data) +// duplicate mount entries. +func matchMountPath(paths []string, currentFile string) (string, error) { + candidate := strings.ReplaceAll(currentFile, "-", "/") + var matches []string + for _, p := range paths { + if p == candidate { + matches = append(matches, p) + } + } + switch len(matches) { + case 1: + return matches[0], nil + case 0: + return "", fmt.Errorf("icarus: %s: no matching file in base pak "+ + "(expected mount path %s, from CurrentFile with '-' converted to '/')", currentFile, candidate) + default: + return "", fmt.Errorf("icarus: %s: ambiguous, matches %v", currentFile, matches) + } +} + +// sanitizeAssetPath validates a bundled asset's mount path before it is +// written into the output pak. .EXMODZ archives are third-party zip files, +// and ParseExmodz (Task 11) carries each entry's raw zip name through +// unchanged as the Assets map key. Without this gate, a crafted entry name +// (a "../" parent traversal, an absolute path, or a Windows drive path) could +// escape the mod's own namespace once the pak is deployed or unpacked +// elsewhere — the pak equivalent of a zip-slip. Rejecting it here, before +// AddFile, keeps that malformed-archive class of input a loud compile +// failure rather than a written-then-discovered problem. +func sanitizeAssetPath(rawZipName string) (string, error) { + normalized := strings.ReplaceAll(rawZipName, `\`, "/") + if strings.Contains(normalized, "\x00") { + return "", fmt.Errorf("icarus: bundled asset %q: contains a NUL byte", rawZipName) + } + if strings.HasPrefix(normalized, "/") || isWindowsDriveAbsolute(normalized) { + return "", fmt.Errorf("icarus: bundled asset %q: absolute paths are not allowed", rawZipName) + } + cleaned := path.Clean(normalized) + if cleaned == "." || cleaned == ".." || strings.HasPrefix(cleaned, "../") { + return "", fmt.Errorf("icarus: bundled asset %q: escapes the mod's own path", rawZipName) + } + return cleaned, nil +} + +// isWindowsDriveAbsolute reports whether p starts with a Windows drive letter +// (e.g. "C:/evil"). Checked on the slash-normalized form, since a zip entry +// written by a Windows tool may carry "C:\evil" — backslashes normalize to +// forward slashes before this check runs. +func isWindowsDriveAbsolute(p string) bool { + return len(p) >= 2 && p[1] == ':' && + ((p[0] >= 'A' && p[0] <= 'Z') || (p[0] >= 'a' && p[0] <= 'z')) +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +```bash +go test ./internal/source/icarus/... -v +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add internal/source/icarus/compile.go internal/source/icarus/compile_test.go +git commit -m "feat: implement exmod compile orchestration (#136)" +``` + +--- + +## Task 13: Wire the compile step into `Service`'s cache-population path + +**Files:** + +- Modify: `internal/domain/game.go:51-78` (`DeployMode`) and `Game` (new `BaseDataPath` field, Step 6b) +- Modify: `internal/storage/config/games.go` (new `data_dump_path` YAML key, Step 6b) +- Modify: `internal/source/source.go` (new `Compiler` optional interface, mirroring `DownloadHeaderProvider`'s type-assertion pattern) +- Modify: `internal/source/icarus/icarus.go` (implement `Compiler`) +- Modify: `internal/core/service.go:455-495` (`DownloadModToCache`'s extract/copy branch) +- Create: `internal/core/service_icarus_compile_test.go` + +**Interfaces:** + +- Consumes: `icarus.Compile` (Task 12), `domain.Game.DeployMode` (existing). +- Produces: `domain.DeployCompile` (new `DeployMode` value), `domain.Game.BaseDataPath` (new optional field), `source.Compiler` interface with `Compile(ctx context.Context, basePakPath, baseDataPath, sourceFilePath, outputPath string) error`. + +- [ ] **Step 1: Add `DeployCompile` to the `DeployMode` enum** + +In `internal/domain/game.go`: + +```go +const ( + DeployExtract DeployMode = iota // Default: extract archives to mod path + DeployCopy // Copy files as-is (for games like Hytale where .zip IS the mod) + DeployCompile // Compile downloaded file into a new artifact before caching (Icarus .exmodz -> .pak) +) + +func (m DeployMode) String() string { + switch m { + case DeployExtract: + return "extract" + case DeployCopy: + return "copy" + case DeployCompile: + return "compile" + default: + return "extract" + } +} + +func ParseDeployMode(s string) DeployMode { + switch s { + case "copy": + return DeployCopy + case "compile": + return DeployCompile + default: + return DeployExtract + } +} +``` + +- [ ] **Step 2: Add the `Compiler` interface to `internal/source/source.go`** + +```go +// Compiler is implemented by sources whose downloaded files need +// transforming into a different artifact before deployment (Icarus's +// .exmodz -> .pak). Service consults it, when DeployMode is DeployCompile, +// after downloading but before committing the file to cache — the result +// replaces the downloaded file in cache, so everything downstream (Install, +// the linker) treats it exactly like a DeployCopy file. +// +// basePakPath and baseDataPath are both resolved by the caller from the game's +// config: basePakPath from game.InstallPath, baseDataPath from the game's +// optional data_dump_path ("" when unset — see Step 6b). sourceFilePath is the +// just-downloaded file; outputPath is where the compiled result must be +// written. +type Compiler interface { + Compile(ctx context.Context, basePakPath, baseDataPath, sourceFilePath, outputPath string) error +} +``` + +- [ ] **Step 3: Implement `Compiler` on `Icarus`** ✅ shipped as `SetDataDir` optional + setter, not a `New` parameter — see below. + +In `internal/source/icarus/icarus.go`, add: + +```go +var _ source.Compiler = (*Icarus)(nil) + +// Compile implements source.Compiler by delegating to the package-level +// Compile function (Task 12) — basePakPath/baseDataPath/sourceFilePath/ +// outputPath map directly onto Compile's basePakPath/localDumpDir/exmodzPath/ +// outputPakPath parameters. The base-table dump store (Task 12a) is supplied +// from the source itself; the per-game dump-directory override arrives as +// baseDataPath, since only the caller has the game's config. +func (s *Icarus) Compile(ctx context.Context, basePakPath, baseDataPath, sourceFilePath, outputPath string) error { + if s.dumps == nil { + return fmt.Errorf("source %q: not initialized with a data directory (SetDataDir was never called)", s.ID()) + } + return Compile(ctx, s.dumps, basePakPath, baseDataPath, sourceFilePath, outputPath) +} +``` + +`Icarus` gains a `dumps *DumpStore` field, nil until wired. **As shipped, this is +NOT constructed in `New`** (the snippet below is what the brief originally proposed): + +```go +// Brief's original proposal — NOT what shipped: +// In Icarus's constructor (Task 8), next to the firestoreClient: + s.dumps = newDumpStore(filepath.Join(dataDir, "icarus", "datadump"), httpClient) +``` + +`New(httpClient, projectID)` was frozen at exactly those two params by Task 8, with +Task 9's `cmd/lmm/root.go` call site already depending on that signature — adding a +third `dataDir` param would mean rewiring `cmd/lmm/root.go`/`root_test.go`, neither of +which is in this task's Files list. Coordinator-approved fix: an optional +post-construction setter, mirroring the existing `SetAPIKey` optional-setter pattern: + +```go +// dumps field, added to the Icarus struct: + dumps *DumpStore // nil until SetDataDir is called + +// SetDataDir wires the base-table dump store's cache directory once the +// service's data directory is known. This is a post-construction setter +// rather than a New parameter because Task 8 froze New(httpClient, projectID) +// at exactly those two params — Task 9's call site already depends on that +// signature — so the data dir arrives the same way API keys do: an optional +// setter the registration pipeline calls when present (cmd/lmm/root.go's +// registerSource, mirroring its existing SetAPIKey wiring). +func (s *Icarus) SetDataDir(dataDir string) { + s.dumps = newDumpStore(filepath.Join(dataDir, "icarus", "datadump"), s.firestore.httpClient) +} +``` + +`cmd/lmm/root.go`'s `registerSource` calls `SetDataDir` via the same optional-setter +type assertion `SetAPIKey` already uses (`dataDir` threaded through +`registerSources`/`registerCustomSources`). A source constructed but never wired +through `registerSource` (or a bare test double) has a nil `dumps`, so `Compile` +fails loudly with the error text above rather than panicking — pinned by +`TestIcarus_Compile_WithoutDataDir_FailsLoudly` and +`TestIcarus_SetDataDir_ConstructsDumpStore` in `icarus_test.go`, plus +`TestRegisterSource_WiresDataDir` in `cmd/lmm/root_test.go`. + +- [ ] **Step 4: Write the failing service-level test** + +Add `internal/core/service_icarus_compile_test.go` in the `core_test` external test package, matching `service_api_source_test.go`'s existing convention (`core.NewService`, `svc.RegisterSource`, `svc.AddGame`, `svc.DownloadMod`, `svc.GetGameCache` — all exported, real entry points confirmed in that file): + +```go +package core_test + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/core" + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/source" + "github.com/stretchr/testify/require" +) + +// fakeCompilerSource is a minimal ModSource that also implements +// source.Compiler, standing in for internal/source/icarus.Icarus (Tasks +// 8/13) without pulling that package into internal/core's tests — this test +// only needs to prove Service invokes Compile when DeployMode is +// DeployCompile, which Task 12 already tests in isolation. +type fakeCompilerSource struct { + downloadURL string + compileCalls int +} + +func (s *fakeCompilerSource) ID() string { return "fake-compiler" } +func (s *fakeCompilerSource) Name() string { return "Fake Compiler Source" } +func (s *fakeCompilerSource) AuthURL() string { return "" } +func (s *fakeCompilerSource) ExchangeToken(ctx context.Context, code string) (*source.Token, error) { + return nil, source.ErrNotSupported +} +func (s *fakeCompilerSource) Search(ctx context.Context, query source.SearchQuery) (source.SearchResult, error) { + return source.SearchResult{}, source.ErrNotSupported +} +func (s *fakeCompilerSource) GetMod(ctx context.Context, gameID, modID string) (*domain.Mod, error) { + return nil, source.ErrNotSupported +} +func (s *fakeCompilerSource) GetDependencies(ctx context.Context, mod *domain.Mod) ([]domain.ModReference, error) { + return nil, source.ErrNotSupported +} +func (s *fakeCompilerSource) GetModFiles(ctx context.Context, mod *domain.Mod) ([]domain.DownloadableFile, error) { + return nil, source.ErrNotSupported +} +func (s *fakeCompilerSource) GetDownloadURL(ctx context.Context, mod *domain.Mod, fileID string) (string, error) { + return s.downloadURL, nil +} +func (s *fakeCompilerSource) CheckUpdates(ctx context.Context, installed []domain.InstalledMod) ([]domain.Update, error) { + return nil, source.ErrNotSupported +} + +// Compile implements source.Compiler by copying the downloaded source file +// through unchanged — this test only asserts Service invoked it with the +// right arguments and used its output, not that it performs real PAK +// compilation (Task 12 covers that). +func (s *fakeCompilerSource) Compile(ctx context.Context, basePakPath, baseDataPath, sourceFilePath, outputPath string) error { + s.compileCalls++ + data, err := os.ReadFile(sourceFilePath) + if err != nil { + return err + } + return os.WriteFile(outputPath, data, 0o644) +} + +var ( + _ source.ModSource = (*fakeCompilerSource)(nil) + _ source.Compiler = (*fakeCompilerSource)(nil) +) + +func TestDownloadMod_DeployCompile_InvokesCompiler(t *testing.T) { + dlSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("fake-exmodz-bytes")) + })) + defer dlSrv.Close() + + installDir := t.TempDir() + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + require.NoError(t, os.WriteFile(basePak, []byte("fake-base-pak"), 0o644)) + + cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} + svc, err := core.NewService(cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + src := &fakeCompilerSource{downloadURL: dlSrv.URL} + svc.RegisterSource(src) + + game := &domain.Game{ID: "icarus", InstallPath: installDir, ModPath: t.TempDir(), DeployMode: domain.DeployCompile} + require.NoError(t, svc.AddGame(game)) + + mod := &domain.Mod{ID: "bear-mount", SourceID: "fake-compiler", GameID: "icarus", Version: "3.3"} + file := &domain.DownloadableFile{ID: "exmodz", FileName: "Bear_Mount.exmodz"} + + result, err := svc.DownloadMod(context.Background(), "fake-compiler", game, mod, file, nil) + require.NoError(t, err) + require.Equal(t, 1, result.FilesExtracted) + require.Equal(t, 1, src.compileCalls) + + gameCache := svc.GetGameCache(game) + require.True(t, gameCache.Exists(game.ID, mod.SourceID, mod.ID, mod.Version)) + files, err := gameCache.ListFiles(game.ID, mod.SourceID, mod.ID, mod.Version) + require.NoError(t, err) + require.Len(t, files, 1) + require.Equal(t, "Bear_Mount_P.pak", files[0]) + + data, err := os.ReadFile(gameCache.GetFilePath(game.ID, mod.SourceID, mod.ID, mod.Version, files[0])) + require.NoError(t, err) + require.Equal(t, "fake-exmodz-bytes", string(data)) +} +``` + +- [ ] **Step 5: Run to verify it fails** + +```bash +go test ./internal/core/... -run TestDownloadMod_DeployCompile -v +``` + +Expected: FAIL (compiler branch doesn't exist yet in `service.go`). + +- [ ] **Step 6: Wire the compile branch into `service.go`** ✅ shipped with an + additional per-file gate — see "Fix round 1" note below. + +In `internal/core/service.go`, modify the block at line 461 (inside the function containing the extract/copy logic shown earlier): + +```go + cachePath, stagePath, err := prepareStaging(gameCache, game, mod) + if err != nil { + return nil, err + } + defer os.RemoveAll(stagePath) //nolint:errcheck + + if game.DeployMode == domain.DeployCompile { + compiler, ok := src.(source.Compiler) + if !ok { + return nil, fmt.Errorf("source %q: game %q requires DeployCompile but source does not implement Compiler", src.ID(), game.ID) + } + basePakPath, err := resolveBasePak(game) + if err != nil { + return nil, err + } + destPath := filepath.Join(stagePath, compiledFileName(file.FileName)) + if err := compiler.Compile(ctx, basePakPath, game.BaseDataPath, archivePath, destPath); err != nil { + return nil, fmt.Errorf("compiling mod: %w", err) + } + if err := commitStagedCache(cachePath, stagePath); err != nil { + return nil, err + } + return &DownloadModResult{FilesExtracted: 1, Checksum: downloadResult.Checksum}, nil + } + + if game.DeployMode == domain.DeployCopy || !s.extractor.CanExtract(archivePath) { + // ...existing copy-mode branch, unchanged... +``` + +Add the two small helpers this references (near the bottom of `service.go`, alongside its other unexported helpers): + +```go +// resolveBasePak locates the currently-installed game's base pak for +// DeployCompile sources. v1 scope: Icarus only, one known pak filename +// pattern — extend this if a second DeployCompile-using game is ever added +// rather than generalizing speculatively now. The relative path below is +// Task 1's empirically-confirmed finding (docs/plans/icarus-pak-format-findings.md), +// recorded before this function was written, not an assumption made here: the +// JSON data tables live in Content/Data/data.pak, NOT in the Content/Paks +// pakchunks, which carry only cooked .uasset/.uexp assets and no JSON at all. +// +// Since rev3 this pak is no longer the source of base table *content* (that +// comes from the hosted dump — Task 12a); it is still required, because it is +// the only authority on which tables the installed game has and on which game +// week is installed. Its parent directory also locates Icarus/Config/version.json. +func resolveBasePak(game *domain.Game) (string, error) { + candidate := filepath.Join(game.InstallPath, "Icarus", "Content", "Data", "data.pak") + if _, err := os.Stat(candidate); err != nil { + return "", fmt.Errorf("locating base pak for %q: %w", game.ID, err) + } + return candidate, nil +} + +// compiledFileName turns a downloaded source filename into the cached +// output's name: same base name, .pak extension, matching Icarus's "_P.pak" +// override convention. +func compiledFileName(sourceFileName string) string { + base := strings.TrimSuffix(sourceFileName, filepath.Ext(sourceFileName)) + return base + "_P.pak" +} +``` + +Add `"strings"` to `service.go`'s imports if not already present (it is, per the existing `strings.EqualFold` call visible at line 450). + +**Fix round 1 (post-review): gate the compile branch per-file, not per-game.** As +shipped, the `if game.DeployMode == domain.DeployCompile {` line above reads +`if game.DeployMode == domain.DeployCompile && isExmodzFile(file.FileName) {`. Review +found that a `DeployCompile` game's catalog can also serve an already-built `.pak` +(`icarus.GetModFiles` enumerates `"pak"` before `"exmodz"`, neither marked primary +when a mod has both) — routing a `.pak` through `Compile` fails loudly (`ParseExmodz` +on non-zip bytes), making plain-pak Icarus mods permanently uninstallable. Fix: a new +helper, placed beside `resolveBasePak`/`compiledFileName`: + +```go +// isExmodzFile reports whether fileName is a compile-eligible archive +// (case-insensitive ".exmodz" suffix). DeployCompile games can also serve +// plain, already-built ".pak" files (icarus.GetModFiles enumerates "pak" +// before "exmodz") - those must NOT be routed through Compile, which expects +// an .exmodz diff (#136 review, Task 13 fix round 1): a prebuilt pak falls +// through to the pre-compile extract/copy logic unchanged, exactly as if +// DeployMode were not DeployCompile at all. +func isExmodzFile(fileName string) bool { + return strings.HasSuffix(strings.ToLower(fileName), ".exmodz") +} +``` + +No config surface changed and no edit was needed to the fallthrough branches — a +`.pak` is not zip/7z/rar, so `!s.extractor.CanExtract` is already true regardless of +`DeployMode`, landing it in the existing copy-as-is branch exactly as on a +`DeployCopy`/`DeployExtract` game. Covered by table-driven tests in +`service_icarus_compile_test.go` +(`TestDownloadMod_DeployCompile_RoutesPerFile`, +`TestDownloadMod_DeployCompile_MixedFileMod`) — see `task-13-report.md`'s "Fix round +1" and "Plan delta" sections for full detail. + +- [ ] **Step 6b: Add the `data_dump_path` game setting (rev4)** + +The local dump-directory override is a per-game path, so it follows the same route +`cache_path` already takes — `games.yaml` → `GameConfig` → `domain.Game`, expanded and +round-tripped on save. Three one-line additions, no new config file and no new loader: + +```go +// internal/storage/config/games.go — GameConfig, beside CachePath: + BaseDataPath string `yaml:"data_dump_path,omitempty"` + +// internal/storage/config/games.go — in the domain.Game literal, beside CachePath: + BaseDataPath: ExpandPath(cfg.BaseDataPath), + +// internal/storage/config/games.go — in the save path's GameConfig literal: + BaseDataPath: game.BaseDataPath, +``` + +```go +// internal/domain/game.go — Game, beside CachePath: + BaseDataPath string // Optional: directory holding an unpacked data.pak JSON + // tree, used instead of fetching the hosted base-table dump (compile games only) +``` + +Documented YAML shape: + +```yaml +games: + icarus: + name: Icarus + install_path: /data/SteamLibrary/steamapps/common/Icarus + mod_path: /data/SteamLibrary/steamapps/common/Icarus/Icarus/Content/Paks + deploy_mode: compile + # Optional. Point at a directory containing an unpacked data.pak JSON tree + # (QuickBMS output, or IMM's extracted "data" folder) to compile from your + # own extraction instead of the hosted community dump. It must match the + # installed game version: it is byte-validated against the game's own pak + # exactly like the hosted dump, and a mismatch is a hard error. + data_dump_path: ~/icarus-data-dump +``` + +`ExpandPath` gives `~` handling for free, matching `cache_path`. + +**No CLI or TUI surface is added, deliberately.** Compiling is pipeline-internal — it runs +inside `DownloadMod`, not as a user-invoked command — so this is configuration, not an +operation, exactly like `cache_path` and `deploy_mode`. Both interfaces already pick it up +by reading `games.yaml`, and neither grows a flag or a screen. This is **not** a CLI/TUI +parity gap: there is no new capability to surface in either. + +Add one loader test alongside the existing games-config tests: + +```go +func TestLoadGames_DataDumpPath(t *testing.T) { + dir := t.TempDir() + yaml := "games:\n icarus:\n name: Icarus\n install_path: /games/icarus\n" + + " mod_path: /games/icarus/mods\n data_dump_path: /dumps/week243\n" + if err := os.WriteFile(filepath.Join(dir, "games.yaml"), []byte(yaml), 0o644); err != nil { + t.Fatal(err) + } + + games, err := LoadGames(dir) + if err != nil { + t.Fatalf("LoadGames: %v", err) + } + if got := games["icarus"].BaseDataPath; got != "/dumps/week243" { + t.Errorf("BaseDataPath = %q, want /dumps/week243", got) + } +} +``` + +- [ ] **Step 7: Run tests to verify they pass** + +```bash +go test ./internal/core/... -run TestDownloadModToCache_DeployCompile -v +go test ./internal/storage/config/... -run TestLoadGames_DataDumpPath -v +go test ./internal/core/... -v +``` + +Expected: PASS, and no regressions in the rest of `internal/core`. + +- [ ] **Step 8: Update the Task 9 README example** + +In `README.md`, change the Icarus `games.yaml` example's `deploy_mode: compile` (was already written that way in Task 9 — verify it matches the now-implemented `ParseDeployMode` string, `"compile"`). + +As shipped, `docs/configuration.md` was also updated (its `deploy_mode` option table and "Deploy Mode" prose only listed `extract`/`copy`, so it was stale on this exact axis) — see `task-13-report.md`. + +- [ ] **Step 9: Full build + vet + test sweep** + +```bash +go build ./... +go vet ./... +go test ./... -v +``` + +Expected: all green. + +- [ ] **Step 10: Commit** + +```bash +git add internal/domain/game.go internal/storage/config/games.go internal/storage/config/games_test.go internal/source/source.go internal/source/icarus/icarus.go internal/core/service.go internal/core/service_icarus_compile_test.go README.md +git commit -m "feat: wire exmod compile step into cache-population pipeline (#136)" +``` + +--- + +## Post-plan manual validation (not automated — do this against your real Icarus install) + +1. **Reader against the real install** (do this as soon as Task 2 is green — it is the + acceptance gate the synthetic fixtures cannot provide): + - `unrealpak.Open` on + `/Icarus/Content/Paks/pakchunk0-WindowsNoEditor.pak` succeeds and `Files()` + returns exactly **9295** entries. + - `unrealpak.Open` on `/Icarus/Content/Data/data.pak` succeeds and `Files()` + returns exactly **298** entries, all ending in `.json`. + - `ReadFile("Factions/D_Factions.json")` on that `data.pak` returns 113 bytes of valid + JSON beginning `{"RowStruct": "/Script/Icarus.Factions"`, and `ReadFile` on an + Oodle-compressed table such as `Items/D_ItemsStatic.json` returns `ErrUnsupportedFormat`. + + Task 1 already verified these numbers against the real files; a mismatch means the index + parser regressed, not that the install differs. + +2. `resolveBasePak`'s path (Task 13, Step 6) is now `Icarus/Content/Data/data.pak` per the + Task 1 spike — confirm it resolves on the target install. + 2b. **Base-table dump fetch + cross-validation** (rev3 — the acceptance gate for Task 12a): + - `detectBuild` on the real install returns the version string in + `Icarus/Config/version.json` (spike 3 saw `3.0.21.155335`, FeatureLevel + `DangerousHorizons`). + - The dump tree downloads unauthenticated from + `https://codeload.github.com/GODOFMINECRAFT4/IcarusData/tar.gz/refs/heads/master` + (~36 MB, a few seconds) and yields several hundred `.json` tables. + - After `LF -> CRLF` restoration, dump tables byte-match the tables the local `data.pak` + stores uncompressed. **Expect this to FAIL until the dump catches up** — at spike time + the install was Week 243 and the dump HEAD was Week 236, giving 3 differing and 6 + missing stored tables. A correct implementation reports that mismatch clearly and + refuses to compile; that is a PASS of the error path, not a bug. + - With a genuinely matching week, `Compile` produces a `_P.pak` whose patched table + differs from the dump's original only in the patched rows. + 2c. **Confirm the dump source is still maintained** before relying on it in a release. It is + a single personal repo (`GODOFMINECRAFT4/IcarusData`, 0 stars, no CI) that has gone + dormant for months before (Dec 2024 → Jul 2025). If it lags persistently, this strategy + needs revisiting — see `docs/plans/icarus-pak-format-findings.md` Part 3. +3. **Confirm the mount point a `_P.pak` needs.** `Writer` stamps `defaultMountPoint` + (`"../../../"`, matching Icarus's pakchunks), but the real `data.pak` uses an absolute + cook-machine path (`C:/BA/work/.../Temp/Data/`). Which one makes the engine mount our + override in the right place is unverified and can only be settled in-game. +4. Confirm the real Firestore project ID (Task 9, Step 1) and update `icarusFirestoreProjectID`. +5. Confirm/flip Firestore security rules to allow public reads on `mods` (design doc's stated assumption — never independently verified in this plan). +6. Run `lmm search icarus "bear"` (or the TUI equivalent) against the real catalog, install a known `.exmodz` mod end-to-end, and confirm Icarus actually loads the resulting `_P.pak` in-game. +7. Revisit the unresolved `-Compress` question from the research spike if the mod's effects don't show up in-game despite a clean compile — that was flagged as unresolved even by experienced modders and is the most likely real-world failure mode this plan's synthetic tests can't catch. Note the Task 1 spike gives this fresh weight: Icarus's own `data.pak` Oodle-compresses 258 of its 298 tables, so an all-stored override pak is _not_ what the engine normally sees. diff --git a/docs/plans/archive/2026-07-29-icarus-exmod-pak-research.md b/docs/plans/archive/2026-07-29-icarus-exmod-pak-research.md new file mode 100644 index 0000000..b0be9bb --- /dev/null +++ b/docs/plans/archive/2026-07-29-icarus-exmod-pak-research.md @@ -0,0 +1,91 @@ +# Icarus `.exmod`/`.exmodz` → PAK Compilation: Research Spike & Design + +**Status:** Design (pre-implementation). Originates from [`IDEAS.md`](../../IDEAS.md) "Specialized Game Mod Support" section; no GitHub issue filed yet — file one before starting implementation per repo workflow. + +**Date:** 2026-07-29 + +## Background + +Icarus is an Unreal Engine game with two mod distribution formats: + +- **`.pak`** — a finished, ready-to-deploy Unreal PAK override file. Deploying these is a solved problem in LMM already (a straight file copy, same shape as the existing `DeployCopy` mode). +- **`.exmodz`** — a zip archive containing an `.EXMOD` JSON manifest (a _diff_ against the base game's data tables) plus, optionally, pre-built `.uasset`/`.uexp` files the mod author already compiled in the Unreal Editor. Turning this into something the game can load requires **building a new `.pak` file on the fly**, which is the actual project: nothing in LMM's pipeline can produce a `.pak` file today, and doing so on Linux (no Unreal Editor, no `UnrealPak.exe` without Wine) is the open question this spec addresses. + +The mod catalog itself (metadata + download URLs) lives in a Firestore database the user maintains, currently consumed only by a server-rendered Rails site ([`project_daedalus`](https://github.com/DonovanMods/project_daedalus)) and a Ruby CLI (`icarus-mod-tools`) via an authenticated service-account keyfile. There is no existing public JSON API. + +## Scope + +**In scope:** end-to-end support for installing an Icarus mod distributed as `.exmodz` — from browsing the catalog through producing and deploying a working `_P.pak`. + +**Explicitly out of scope for this spec:** + +- Any other Unreal-Engine game (Satisfactory, etc.) — see [Future Potential](#future-potential-not-designed-now). +- Compression or encryption support in the PAK writer beyond the minimum needed for Icarus (uncompressed, unencrypted). +- Redistributing Epic's `UnrealPak.exe` (unlike IMM) or requiring Wine. + +## Research Findings + +Grounded in a real sample (`Bear_Mount.EXMODZ`), the `icarus-mod-tools`/`project_daedalus` repos, and public modding-community sources — not speculation: + +- **Engine/format**: Icarus runs UE 4.26.2/4.27, classic `.pak` format (not UE5 IoStore). This is the best-case format to target: well-documented, well-tooled elsewhere. +- **No encryption**: community guides routinely unpack `data.pak` with plain `UnrealPak.exe`; no AES-key-extraction step appears anywhere in the modding community's process. +- **Confirmed prior art**: Jimk72's [`Icarus_Software`](https://github.com/Jimk72/Icarus_Software) repo (the actual IMM source) ships a redistributed `UnrealPak.zip` alongside `DUMP_Week_140.zip` — a pre-extracted, per-week-build cache of the base game's data files. This confirms the `compatibility: "w57"`-style field used throughout the mod catalog is Icarus's own weekly build numbering, and confirms IMM's actual pipeline: unpack base data (from a cached dump) → apply diff → repack with `UnrealPak.exe`. +- **`.EXMOD` diff shape** (from the real sample): a JSON document with mod metadata (`name`, `author`, `version`, `description`, ...) and a `Rows` array of `{"CurrentFile": "", "File_Items": [{"Name": "", }]}`. This patches **plain JSON data tables already shipped inside the game's PAK** — not compiled binary assets, and not anything requiring an Unreal "cook" step. +- **`.EXMODZ` bundled assets**: the same sample also ships finished `.uasset`/`.uexp` pairs (new skeletal meshes, animations, blueprints, icons) that the mod author already compiled externally. These need placing into the output PAK at the correct path, not compiling. +- **Linux-native tooling precedent**: [`repak`](https://github.com/trumank/repak) (Rust) reads/writes this exact PAK version range natively on Linux with no Wine or Epic binary — proof the underlying problem is tractable on Linux, even though LMM won't shell out to it (see [Package Layout](#package-layout)). +- **No mature Go PAK library**: [`pakr`](https://github.com/recogni/pakr) exists but is WIP/limited — LMM needs a small purpose-built reader/writer, not an off-the-shelf dependency. +- **Unresolved even by experienced modders**: a public GitHub issue ([masterj1337/IcarusMods#1](https://github.com/masterj1337/IcarusMods/issues/1)) shows a modder packing a `.pak` correctly with uncertainty over whether `-Compress` is required, and the result not taking effect in-game. Exact packing parameters are not fully settled community knowledge — this needs direct empirical validation, not assumption. + +## Proposed Architecture + +### Icarus ModSource (catalog access) + +A new built-in `ModSource` at `internal/source/icarus/`, structurally parallel to `internal/source/nexusmods/` and `internal/source/curseforge/` (hand-written Go client, registered in `Service.RegisterSource()` like any other built-in) — **not** a YAML `custom` source, since Firestore's typed-value REST document format (`{fields: {name: {stringValue: "..."}}}`) doesn't fit `custom.API`'s flat-JSON dot-path field mapping. + +- Reads Firestore's public REST API directly (`https://firestore.googleapis.com/v1/projects/{project}/databases/(default)/documents/...`) with **no credentials** — confirmed the `mods` collection will have public read rules. +- No `cloud.google.com/go/firestore` SDK dependency (gRPC/protobuf, heavy) — plain `net/http` plus a small typed-value decoder, same dependency weight as the existing NexusMods/CurseForge clients. +- `Search` mirrors `project_daedalus`'s own approach: Firestore's simple REST surface doesn't support server-side text queries the way NexusMods/CurseForge do, so the source fetches the `mods` collection (paginated) and filters client-side by name/author/description — exactly what `ModsController#find_mods` already does today. +- Field mapping, per `modinfo.json.template.md` v2: `name`, `author`, `version`, `compatibility` (Icarus week-build string), `description`, `files.pak` / `files.exmodz` (direct download URLs — GitHub raw links in practice), `imageURL`, `readmeURL`. `GameID` is fixed to `"icarus"` (single-game database). +- `GetDownloadURL` returns the stored URL directly — no signing/redirect dance needed, closer in shape to `custom.Manifest`/`custom.Directory` than to NexusMods' OAuth flow. + +### Compile pipeline (the core of this project) + +Triggered when the selected downloadable file is an `.exmodz`, positioned as a new pre-deploy stage between "cache has final source files" and the existing `Linker.Deploy`: + +1. **Base data comes from the local install, not a hosted dump.** LMM reads whatever `data.pak` is actually present in the user's installed game right now, rather than replicating IMM's per-week `DUMP_Week_N.zip` cache (which depends on someone continuing to host those dumps indefinitely). This always diffs against ground truth and is more robust than IMM's own approach. +2. **Targeted extraction** — parse just the PAK index (not the full multi-GB archive: `repak`'s own docs note it "only parses index initially, reads file data upon request," and the Go reader should follow the same shape), then extract only the specific files each exmod's `Rows[].CurrentFile` entries reference. +3. **Apply the diff** — pure JSON manipulation: for each referenced base file, merge in `File_Items[].Name`-keyed field overrides. No Unreal-specific tooling needed for this step. +4. **Assemble the mod layer** — patched JSON files at their PAK-internal paths, plus any `.uasset`/`.uexp` files already bundled in the `.EXMODZ`, placed as-is. +5. **Write a new override `_P.pak`** via the Go PAK writer (uncompressed/unencrypted to start — see [Open Risks](#open-risks)), following the community's established `_P.pak` → `Content/Paks/mods` convention. +6. **Deploy** via the existing `Linker` unchanged — at this point it's a single finished file, same shape as `DeployCopy`. + +### Package layout + +- `internal/unrealpak/` — game-agnostic PAK index reader + writer for the UE 4.25–4.27 unencrypted format range. Zero Icarus-specific knowledge. This is the piece a future Satisfactory (or other UE game) effort could potentially reuse. +- `internal/source/icarus/` — the Firestore `ModSource`, the `.EXMOD` diff-application logic (JSON row merging), and `.EXMODZ` → PAK-path mapping. Uses `internal/unrealpak/` but owns all Icarus-specific knowledge. +- Deploy pipeline: the compile step needs a new hook point that doesn't exist today (`Linker.Deploy` is strictly 1:1 file placement, and `Game.DeployMode` is a closed `Extract`/`Copy` enum). Proposed shape: an optional `Compiler` capability a source/game can provide (mirroring the existing `CapabilityReporter`/`TypeLabeler` type-assertion pattern already used for optional `ModSource` behavior), invoked between cache population and `Linker.Deploy` when present. Exact interface and wiring is an implementation-plan decision, not finalized here. + +### Error handling + +No silent fallbacks. If the installed `data.pak`'s version/compression/encryption doesn't match what the reader expects, or a `Rows[].CurrentFile` target isn't found in the base data, this fails loudly with a clear, actionable error — consistent with this repo's existing fail-fast precedent (#95). + +### Testing + +Real Icarus game files cannot ship as test fixtures (not ours to redistribute). The reader/writer gets round-trip tests in CI (write a synthetic PAK with our own writer, read it back, assert index/content correctness byte-for-byte) plus diff-application unit tests against a synthetic base JSON + a real `.EXMOD` sample's `Rows` shape. Validation against a genuine `data.pak` happens manually, locally, outside the automated suite — and is also where the format assumptions below get confirmed or falsified. + +## Open Risks + +- **Everything above is inferred from community discussion, not verified against real bytes.** Before writing any Go PAK code, the concrete next step is validating the actual format against a real local `data.pak`: confirm the PAK version/footer magic, confirm zero encryption in practice, and confirm the `.EXMODZ` internal folder structure (`ASS/`, `BP/`, ...) maps directly to Icarus's in-PAK mount paths with no translation layer. +- The unresolved `-Compress` community issue suggests packing parameters matter in ways not fully settled even by experienced modders — plan to empirically test both compressed and uncompressed output against a real local install before committing to one. +- This is a multi-week effort with no existing library to lean on for the hard part (no mature Go PAK library) — implementation-plan sizing should reflect that, not treat it as a quick add-on. + +## Future Potential (not designed now) + +The user has flagged that this may lay groundwork for supporting other Unreal-Engine-based games (Satisfactory is explicitly mentioned in `IDEAS.md`, requiring its own separate tooling — "Satisfactory Mod Loader," ficsit.app API). Keeping `internal/unrealpak/` free of Icarus-specific assumptions is a deliberate, low-cost choice that keeps that door open. This is _not_ a commitment to build Satisfactory support, and no Satisfactory-specific design work has been done — a future effort there would need its own research spike (different UE version, different mod-loader mechanism, entirely unverified). + +## Out of Scope Recap + +- Encrypted or IoStore PAK support. +- Any non-Icarus game. +- Redistributing Epic's `UnrealPak.exe` or requiring Wine/Proton. +- Auth/write access to Firestore (read-only, public). diff --git a/docs/plans/archive/2026-08-01-icarus-merged-pak.md b/docs/plans/archive/2026-08-01-icarus-merged-pak.md new file mode 100644 index 0000000..fe9796f --- /dev/null +++ b/docs/plans/archive/2026-08-01-icarus-merged-pak.md @@ -0,0 +1,3299 @@ +# Icarus Merged-Pak Compilation Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace per-mod `.exmodz` compilation for `deploy_mode: compile` games with a merged-ONLY model: every enabled `.exmodz` mod's row-level table diffs are applied sequentially, in profile load order, against the game's base tables into ONE profile-level `zzz_LMM_Merged_P.pak`, so multiple table-patching mods compose (field-level merge) instead of one mod's whole-table pak silently shadowing another's. + +**Architecture:** A new `source.MergeCompiler` interface (replacing #196's `source.Compiler`) gives Icarus a `MergeCompile(sources []MergeSource, ...)` entry point that threads each mod's `.EXMOD` row-upserts through the SAME evolving table bytes — `ApplyRowPatch`'s existing shallow-merge semantics do the actual composing, unchanged. Ingest (download/import) drops per-mod pak generation entirely: it now only validates the `.exmodz` and retains its bytes in cache (zero deployment members). The merged pak itself is tracked as a synthetic, profile-scoped "mod" (`sourceID="lmm-merged"`, `modID="merged-pak"`) that reuses the EXISTING `Installer.Install`/`Uninstall`/cache/deployed-file machinery verbatim — no schema changes. A new `Service.syncMergedPak` (fingerprint-gated, cheap when nothing changed) is called from every mutation flow that can change the enabled-mod set, load order, mod version, or base pak; `lmm update`/`lmm verify` are the safety net that catches anything a hook call site misses, exactly as #196 relies on for base-pak drift today. + +**Tech Stack:** Go stdlib only (`encoding/json` for the fingerprint marker, `crypto/md5` — already used via `md5File` — for exmodz checksums); no new dependencies. Builds on #196's staging/atomic-commit/reserved-marker cache primitives and #136/#175's `internal/unrealpak` reader/writer. + +## Global Constraints + +- Standard library only — no new third-party dependencies (matches `~/.claude/GO.md` and this repo's existing zero-new-deps discipline). +- Fail loud: a malformed `.exmodz`, an unresolvable base-pak table reference, or a merge failure must return an actionable error, never silently skip or produce a partial/incorrect merged pak. +- CLI and TUI parity through `internal/core` — no logic duplicated between `cmd/lmm` and `internal/tui`; both call the same `Service` methods. +- `--json` contract changes are additive-only: existing fields keep their exact names/types/omit-empty behavior; only new optional fields may be added. +- TDD: every task starts with a failing test before the implementation that makes it pass (`- [ ] Step: Write the failing test` / `- [ ] Step: Run it, confirm it fails` / `- [ ] Step: Implement` / `- [ ] Step: Run it, confirm it passes` / `- [ ] Step: Commit`). +- `gofmt`, `go vet`, `go test ./...`, and `trunk check` must be clean at the end of every task's commit. +- CHANGELOG discipline: this plan AMENDS the existing `[Unreleased] / Added` bullet for #196 in place (that bullet has never shipped in a tagged release — see Task 14) rather than adding a second, overlapping entry. +- Plain (non-`.exmodz`) `.pak` mods and non-`DeployCompile` games must be byte-for-byte unaffected by every task in this plan. +- Every new/changed function gets a doc comment stating the _why_, matching this repo's existing density (see `internal/core/updater.go`, `internal/storage/cache/cache.go` for the house style) — not the terser style used in this plan's own code samples. + +## Design Decisions (locked in; one flagged for coordinator confirmation) + +These were resolved by direct investigation of this repository (deployed-file schema, deploy trigger points, exmod/exmodz format) and, for the merge algorithm's core hypothesis, by **extraction-verification**: the merge engine and the fingerprint-equality logic below were written and tested as real, runnable Go code against a scratch copy of `develop` tip `541b485` before this plan was finalized. See each task's "Extraction-verified" note. + +1. **Merged pak filename: `zzz_LMM_Merged_P.pak`.** UE's pak platform file mounts paks within a directory in filename-sort order, and a later-mounted pak wins same-path conflicts (this repo's own `icarusContentMountPoint` doc comment already notes "UE orders paks by its own filename-sort rules within a directory," and the issue body flags the same point). `zzz` forces last-alphabetical mount (a long-standing UE-modding convention for "load last, highest priority" — used so the merged pak's authoritative combined table state can never be silently shadowed by a plain prebuilt `.pak` mod that happens to also carry a table override). `LMM` makes the file greppable/recognizable as lmm-owned (useful for support and for the ownership design in Task 5). `Merged` names its content. `_P` matches the existing UE override-pak suffix convention this codebase already uses (`compiledFileName`, `internal/core/service.go:1008`). + +2. **Merged pak identity for cache/deploy tracking: `sourceID = domain.SourceMerged = "lmm-merged"`, `modID = "merged-pak"`, cache `version = "merged"`.** The `deployed_files` table's `source_id`/`mod_id` columns are `NOT NULL` with no existing owner-less concept (verified: `internal/storage/db/migrations.go` `migrateV7`, `internal/storage/db/files.go`). Rather than a schema migration, the merged pak is tracked as a synthetic, singleton "mod" per `(game, profile)` — this reuses `Installer.Install`/`Uninstall`/`cache.Cache` verbatim (Task 5/6), inherits the SAME `deployed_files` ownership and #168-class residue risk as every other deployed file (not a new, worse risk class), and needs zero schema changes. `domain.SourceMerged` follows the existing `domain.SourceLocal = "local"` sentinel-string precedent (`internal/domain/mod.go:20`) — same acceptance of the (already-accepted) theoretical collision risk with a user-named custom source. + +3. **Locked-mod semantics — PROPOSED, flagged for coordinator confirmation:** a locked mod's retained `.exmodz` diff STILL participates in every re-merge, at its locked version, unchanged. A lock does NOT exclude the mod from the merge and does NOT freeze the whole merged pak. Locking only prevents THAT mod's own version from advancing (the existing, #196-established meaning of "lock-wins" — `ApplyRecompile`'s `ErrModLocked` gate refuses to change what a locked mod's OWN cache/retained-source content is, but #196 already established that a locked mod's _existing_ diff is still reapplied when the BASE PAK changes; this plan simply extends that same reasoning to "when anything else in the profile changes, not just the base pak"). Freezing the whole merge on any lock present would make locking one mod block every OTHER mod's changes from ever reaching the deployed game — directly contradicting the purpose of a per-mod lock, and a severe UX regression for a feature that is supposed to make multi-mod profiles WORK. The merged pak is a separate, profile-level artifact (not "the locked mod's own files") — reading a locked mod's retained source to feed the merge is not "touching" it in the sense `ApplyRecompile`'s lock gate protects against (which is about REWRITING a locked mod's own cache/version content, not READING it). **This is the one item in this plan the coordinator should explicitly confirm before implementation starts** (Task 13 is a dedicated, isolated test task for exactly this behavior, so confirming/reversing it later is a small, contained change). + +4. **`CheckGameUpdates` gains a `profileName` parameter.** #196's `Service.CheckGameUpdates(ctx, game, installed)` has no way to know which profile's merged pak to check (staleness is profile-scoped, `installed` alone doesn't reliably carry it). All 4 existing call sites (`cmd/lmm/update.go` ×2, `internal/tui/service_core.go` ×2) are updated in Task 9 — internal signature change, not user-facing. + +5. **Per-mod `.exmodz` "fingerprint" simplifies to "retained + validated," full stop.** #196's per-mod `MarkBaseIndexHash`/`BaseIndexHashes` cache markers (recording a base-pak IndexHash per compiled FILE) become dead code once there is no per-mod compile output to fingerprint — the MERGED pak's fingerprint (Task 5) subsumes that job at the profile level, keyed by each contributing file's own content checksum (`md5File` over the retained `.exmodz` bytes, computed on demand — these files are small, this repo's own research established the largest real base table is 7.3 MB, so re-hashing on every staleness check is cheap and avoids a second marker to keep in sync). Task 4 removes the now-dead #196 functions. + +## File Structure + +| File | Responsibility | +| --------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `internal/source/source.go` | `MergeCompiler` interface + `MergeSource` type (replaces `Compiler`) | +| `internal/source/icarus/merge.go` (new) | `MergeCompile` — the merge engine — and `ValidateSource` | +| `internal/source/icarus/icarus.go` | `*Icarus` implements `MergeCompiler` instead of `Compiler` | +| `internal/core/service.go` | Ingest simplification (download + local-ingest DeployCompile branches); removal of dead #196 per-mod compile helpers | +| `internal/core/importer.go` | Ingest simplification (import DeployCompile branch) | +| `internal/core/merged_pak.go` (new) | `MergedFingerprint` type, marker read/write, `enabledExmodzSources`, `Service.syncMergedPak`, `Service.ApplyMergedPakRegen` | +| `internal/core/updater.go` | `CheckGameUpdates` signature change; removal of `CheckBaseStaleness`/`ApplyRecompile`/`ClassifyRetainedSourceStatError` (superseded by `merged_pak.go`) | +| `internal/core/flows.go` | `syncMergedPak` hook calls in `EnableMod`, `DisableMod`, `UninstallMod`, `DeployProfile`, `ApplyProfileSwitch`, `ApplyUpdate`, `ApplyInstall`; new `Service.ReorderProfileMods` | +| `cmd/lmm/update.go` | `CheckGameUpdates` call sites updated; synthetic merged-pak row rendering (table + `--json`); apply dispatch | +| `cmd/lmm/verify.go` | Replace per-mod `stale_compile` pre-pass with the profile-level merged-pak check | +| `cmd/lmm/profile.go` | `pm.ReorderMods` call site switched to `Service.ReorderProfileMods` | +| `internal/tui/service_core.go` | Mirrors `cmd/lmm/update.go`'s wiring; `ReorderMods` switched to `Service.ReorderProfileMods` | +| `internal/tui/actions_provider.go` | No field changes needed (`UpdateItem.RecompileNeeded`/`VersionLabel()` already generic) | +| `CHANGELOG.md` | Amend the unshipped #196 `[Unreleased]` bullet | + +--- + +### Task 1: `source.MergeCompiler` interface + `icarus.MergeCompile` merge engine + +**Files:** + +- Modify: `internal/source/source.go:156-158` (replace `Compiler` interface) +- Create: `internal/source/icarus/merge.go` +- Test: `internal/source/icarus/merge_test.go` +- Modify: `internal/source/icarus/icarus.go` (implement `MergeCompiler` instead of `Compiler`) +- Test: `internal/source/icarus/icarus_test.go` (interface assertion) + +**Interfaces:** + +- Consumes: `internal/source/icarus/exmod.go`'s `ParseExmod`, `ApplyRowPatch`, `ExmodDiff`, `ExmodRow` (unchanged); `exmodz.go`'s `ParseExmodz`, `ExmodzBundle` (unchanged); `compile.go`'s `resolveCurrentFile`, `sanitizeAssetPath`, `icarusDataTablePrefix`, `icarusContentMountPoint`, `endOfModSentinel` (unchanged, package-private, reused as-is); `internal/unrealpak`'s `Open`, `Create`, `WithMountPoint`, `Reader.ReadFile`, `Reader.Files` (unchanged). +- Produces: `source.MergeCompiler` interface (consumed by Task 2/3's `mergeCompilerSourceForGame`); `icarus.MergeSource{ModRef, ExmodzPath string}`; `icarus.MergeCompile(basePakPath string, sources []MergeSource, outputPakPath string) (warnings []string, err error)`; `icarus.ValidateSource(exmodzPath string) error`. + +**Extraction-verified:** this task's `MergeCompile` body below is the EXACT code verified against a scratch copy of `develop` tip `541b485` (`/tmp/.../scratchpad/lmm-extract-verify`) with 6 passing tests proving: (a) two mods patching DIFFERENT fields of the SAME row both survive (the crux of #197 — whole-pak last-wins would lose one), (b) two mods patching DIFFERENT tables both land in one merged pak, (c) two mods patching the SAME field of the SAME row correctly last-wins with no special handling needed, (d) same-path bundled ASSET collisions last-win AND return a warning naming both mods, (e) a content-adding mod (new row) composes correctly with a table-patching mod, (f) the N=1 case (single enabled mod) produces byte-identical table content to the existing `Compile()`. No code changes were needed after the first defect fix below. + +- [ ] **Step 1: Write the failing tests** + +Create `internal/source/icarus/merge_test.go`: + +```go +package icarus + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/source" + "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" +) + +// TestMergeCompile_FieldLevelMergeAcrossMods is the crux of #197: two mods +// patch DIFFERENT fields of the SAME row in the SAME table. Whole-pak +// last-wins (the #136 status quo) would lose one mod's field entirely; +// sequential upserts must preserve BOTH. +func TestMergeCompile_FieldLevelMergeAcrossMods(t *testing.T) { + baseTables := map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Rows":[{"Name":"Mount_Bear","BaseMovementSpeed":200,"BaseHealth":500}]}`), + } + basePak := writeTestBasePak(t, baseTables) + + modA := writeTestExmodzFile(t, `{"name":"Speed Mod","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":235}]}]}`, nil) + modB := writeTestExmodzFile(t, `{"name":"Health Mod","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseHealth":800}]}]}`, nil) + + outputPath := filepath.Join(t.TempDir(), "merged_P.pak") + warnings, err := MergeCompile(context.Background(), basePak, []source.MergeSource{ + {ModRef: "icarus:speed-mod", ExmodzPath: modA}, + {ModRef: "icarus:health-mod", ExmodzPath: modB}, + }, outputPath) + if err != nil { + t.Fatalf("MergeCompile: %v", err) + } + if len(warnings) != 0 { + t.Errorf("warnings = %v, want none (no asset collision in this fixture)", warnings) + } + + r, err := unrealpak.Open(outputPath) + if err != nil { + t.Fatalf("opening merged output: %v", err) + } + defer r.Close() //nolint:errcheck + + merged, err := r.ReadFile("data/AI/D_AIGrowth.json") + if err != nil { + t.Fatalf("ReadFile merged data table: %v", err) + } + if !bytes.Contains(merged, []byte(`"BaseMovementSpeed":235`)) { + t.Errorf("merged table = %s, want BaseMovementSpeed 235 (mod A's field) to survive", merged) + } + if !bytes.Contains(merged, []byte(`"BaseHealth":800`)) { + t.Errorf("merged table = %s, want BaseHealth 800 (mod B's field) to survive", merged) + } +} + +// TestMergeCompile_DifferentTablesFromDifferentMods proves the OTHER +// whole-pak-last-wins failure mode (#197's issue body point 1): mod A +// patches table X, mod B patches table Y - both must land in the single +// merged pak, not just the last mod's table. +func TestMergeCompile_DifferentTablesFromDifferentMods(t *testing.T) { + baseTables := map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Rows":[{"Name":"Mount_Bear","BaseMovementSpeed":200}]}`), + "Items/D_ItemsStatic.json": []byte(`{"Rows":[{"Name":"Item_Saddle","Weight":5}]}`), + } + basePak := writeTestBasePak(t, baseTables) + + modA := writeTestExmodzFile(t, `{"name":"Mount Mod","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":300}]}]}`, nil) + modB := writeTestExmodzFile(t, `{"name":"Item Mod","Rows":[{"CurrentFile":"Items-D_ItemsStatic.json","File_Items":[{"Name":"Item_Saddle","Weight":1}]}]}`, nil) + + outputPath := filepath.Join(t.TempDir(), "merged_P.pak") + if _, err := MergeCompile(context.Background(), basePak, []source.MergeSource{ + {ModRef: "icarus:mount-mod", ExmodzPath: modA}, + {ModRef: "icarus:item-mod", ExmodzPath: modB}, + }, outputPath); err != nil { + t.Fatalf("MergeCompile: %v", err) + } + + r, err := unrealpak.Open(outputPath) + if err != nil { + t.Fatalf("opening merged output: %v", err) + } + defer r.Close() //nolint:errcheck + + aiTable, err := r.ReadFile("data/AI/D_AIGrowth.json") + if err != nil { + t.Fatalf("ReadFile AI table: %v", err) + } + if !bytes.Contains(aiTable, []byte(`"BaseMovementSpeed":300`)) { + t.Errorf("AI table = %s, want mod A's patch", aiTable) + } + itemsTable, err := r.ReadFile("data/Items/D_ItemsStatic.json") + if err != nil { + t.Fatalf("ReadFile Items table: %v", err) + } + if !bytes.Contains(itemsTable, []byte(`"Weight":1`)) { + t.Errorf("Items table = %s, want mod B's patch", itemsTable) + } +} + +// TestMergeCompile_SameRowSameField_LastWins pins the EXPECTED (not +// warned-about) outcome when two mods genuinely conflict on the exact same +// field of the exact same row: later-in-order wins, ordinary upsert +// semantics, no special handling needed. +func TestMergeCompile_SameRowSameField_LastWins(t *testing.T) { + baseTables := map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Rows":[{"Name":"Mount_Bear","BaseMovementSpeed":200}]}`), + } + basePak := writeTestBasePak(t, baseTables) + + modA := writeTestExmodzFile(t, `{"name":"A","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":300}]}]}`, nil) + modB := writeTestExmodzFile(t, `{"name":"B","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":400}]}]}`, nil) + + outputPath := filepath.Join(t.TempDir(), "merged_P.pak") + if _, err := MergeCompile(context.Background(), basePak, []source.MergeSource{ + {ModRef: "icarus:a", ExmodzPath: modA}, + {ModRef: "icarus:b", ExmodzPath: modB}, + }, outputPath); err != nil { + t.Fatalf("MergeCompile: %v", err) + } + + r, err := unrealpak.Open(outputPath) + if err != nil { + t.Fatalf("opening merged output: %v", err) + } + defer r.Close() //nolint:errcheck + merged, err := r.ReadFile("data/AI/D_AIGrowth.json") + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if !bytes.Contains(merged, []byte(`"BaseMovementSpeed":400`)) { + t.Errorf("merged table = %s, want mod B's (later, order-2) value 400 to win", merged) + } + if bytes.Contains(merged, []byte(`"BaseMovementSpeed":300`)) { + t.Errorf("merged table = %s, mod A's value should have been overwritten", merged) + } +} + +// TestMergeCompile_AssetCollision_LastWinsWithWarning: two mods bundle a +// prebuilt asset at the SAME path - cannot compose like a table row, so +// last-applied wins AND a warning is returned. +func TestMergeCompile_AssetCollision_LastWinsWithWarning(t *testing.T) { + basePak := writeTestBasePak(t, map[string][]byte{"AI/D_AIGrowth.json": []byte(`{"Rows":[]}`)}) + + modA := writeTestExmodzFile(t, `{"name":"A","Rows":[]}`, map[string][]byte{ + "Shared/ASS/SK_Shared.uasset": []byte("from-mod-a"), + }) + modB := writeTestExmodzFile(t, `{"name":"B","Rows":[]}`, map[string][]byte{ + "Shared/ASS/SK_Shared.uasset": []byte("from-mod-b"), + }) + + outputPath := filepath.Join(t.TempDir(), "merged_P.pak") + warnings, err := MergeCompile(context.Background(), basePak, []source.MergeSource{ + {ModRef: "icarus:a", ExmodzPath: modA}, + {ModRef: "icarus:b", ExmodzPath: modB}, + }, outputPath) + if err != nil { + t.Fatalf("MergeCompile: %v", err) + } + if len(warnings) != 1 { + t.Fatalf("warnings = %v, want exactly 1 asset-collision warning", warnings) + } + if !bytes.Contains([]byte(warnings[0]), []byte("Shared/ASS/SK_Shared.uasset")) { + t.Errorf("warning = %q, want it to name the colliding path", warnings[0]) + } + if !bytes.Contains([]byte(warnings[0]), []byte("icarus:b")) { + t.Errorf("warning = %q, want it to name the winning mod", warnings[0]) + } + + r, err := unrealpak.Open(outputPath) + if err != nil { + t.Fatalf("opening merged output: %v", err) + } + defer r.Close() //nolint:errcheck + asset, err := r.ReadFile("Shared/ASS/SK_Shared.uasset") + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(asset) != "from-mod-b" { + t.Errorf("asset content = %q, want mod B's (later-applied) content to win", asset) + } +} + +// TestMergeCompile_ContentAddingModComposesWithPatchMod: one mod ADDS a +// brand-new row (a new mountable species), another PATCHES an existing row +// in the SAME table. Both must survive in the merged output. +func TestMergeCompile_ContentAddingModComposesWithPatchMod(t *testing.T) { + baseTables := map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Rows":[{"Name":"Mount_Bear","BaseMovementSpeed":200}]}`), + } + basePak := writeTestBasePak(t, baseTables) + + patchMod := writeTestExmodzFile(t, `{"name":"Patch","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":250}]}]}`, nil) + addMod := writeTestExmodzFile(t, `{"name":"NewSpecies","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Wolf","BaseMovementSpeed":320}]}]}`, nil) + + outputPath := filepath.Join(t.TempDir(), "merged_P.pak") + if _, err := MergeCompile(context.Background(), basePak, []source.MergeSource{ + {ModRef: "icarus:patch", ExmodzPath: patchMod}, + {ModRef: "icarus:add", ExmodzPath: addMod}, + }, outputPath); err != nil { + t.Fatalf("MergeCompile: %v", err) + } + + r, err := unrealpak.Open(outputPath) + if err != nil { + t.Fatalf("opening merged output: %v", err) + } + defer r.Close() //nolint:errcheck + merged, err := r.ReadFile("data/AI/D_AIGrowth.json") + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if !bytes.Contains(merged, []byte(`"BaseMovementSpeed":250`)) { + t.Errorf("merged table = %s, want the patched Mount_Bear speed", merged) + } + if !bytes.Contains(merged, []byte(`"Mount_Wolf"`)) { + t.Errorf("merged table = %s, want the newly-added Mount_Wolf row", merged) + } +} + +// TestMergeCompile_SingleSource_MatchesCompile proves the N=1 degenerate +// case (a profile with exactly one enabled exmodz mod) produces byte- +// identical table content to the existing single-mod Compile() - the +// merged-only model must not regress the already-shipped single-mod path. +func TestMergeCompile_SingleSource_MatchesCompile(t *testing.T) { + baseTables := map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Rows":[{"Name":"Mount_Bear","BaseMovementSpeed":200}]}`), + } + basePak := writeTestBasePak(t, baseTables) + manifest := `{"name":"Bear Mount","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":235}]}]}` + exmodzPath := writeTestExmodzFile(t, manifest, map[string][]byte{ + "Bear_Mount/ASS/ITM/SK_ITM_Saddle_Bear.uasset": []byte("fake-asset"), + }) + + compileOut := filepath.Join(t.TempDir(), "compile_P.pak") + if err := Compile(basePak, exmodzPath, compileOut); err != nil { + t.Fatalf("Compile: %v", err) + } + mergeOut := filepath.Join(t.TempDir(), "merge_P.pak") + if _, err := MergeCompile(context.Background(), basePak, []source.MergeSource{{ModRef: "icarus:bear-mount", ExmodzPath: exmodzPath}}, mergeOut); err != nil { + t.Fatalf("MergeCompile: %v", err) + } + + cr, err := unrealpak.Open(compileOut) + if err != nil { + t.Fatalf("opening Compile output: %v", err) + } + defer cr.Close() //nolint:errcheck + mr, err := unrealpak.Open(mergeOut) + if err != nil { + t.Fatalf("opening MergeCompile output: %v", err) + } + defer mr.Close() //nolint:errcheck + + cTable, err := cr.ReadFile("data/AI/D_AIGrowth.json") + if err != nil { + t.Fatalf("Compile ReadFile: %v", err) + } + mTable, err := mr.ReadFile("data/AI/D_AIGrowth.json") + if err != nil { + t.Fatalf("MergeCompile ReadFile: %v", err) + } + if !bytes.Equal(cTable, mTable) { + t.Errorf("Compile table = %s, MergeCompile table = %s, want identical for N=1", cTable, mTable) + } +} + +// TestValidateSource_ValidExmodz_NoError proves ValidateSource accepts a +// well-formed .exmodz without compiling anything (no basePak needed). +func TestValidateSource_ValidExmodz_NoError(t *testing.T) { + exmodzPath := writeTestExmodzFile(t, `{"name":"OK","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":200}]}]}`, nil) + if err := ValidateSource(exmodzPath); err != nil { + t.Errorf("ValidateSource: %v, want nil for a well-formed .exmodz", err) + } +} + +// TestValidateSource_MalformedExmodz_Errors proves a corrupt/unparseable +// .exmodz fails loud at validate time (ingest-time), not silently deferred +// to the next merge. +func TestValidateSource_MalformedExmodz_Errors(t *testing.T) { + path := filepath.Join(t.TempDir(), "bad.exmodz") + if err := os.WriteFile(path, []byte("not a zip file"), 0o644); err != nil { + t.Fatal(err) + } + if err := ValidateSource(path); err == nil { + t.Error("ValidateSource: got nil error, want a failure for a non-zip file") + } +} +``` + +- [ ] **Step 2: Run the tests, confirm they fail to compile** + +Run: `go test ./internal/source/icarus/... -run 'TestMergeCompile|TestValidateSource' -v` +Expected: build failure — `undefined: MergeCompile`, `source.MergeSource` undefined (the `MergeCompiler` interface doesn't exist in `internal/source` yet), `undefined: ValidateSource`. + +- [ ] **Step 3: Add the `MergeCompiler` interface to `internal/source/source.go`** + +Read `internal/source/source.go:142-158` first (the `DownloadHeaderProvider` and `Compiler` definitions) to match the file's exact comment style, then replace the `Compiler` interface block (`internal/source/source.go:156-158`) with: + +```go +// MergeCompiler is implemented by sources whose compile-eligible files must +// be merged across every enabled mod into ONE profile-level artifact rather +// than compiled per-mod (#197: Icarus's cross-mod table merge - a whole-pak +// last-wins deploy would silently drop one mod's table rows whenever two +// mods patch the same table). Replaces #196's Compiler interface, which +// this source no longer implements: there is no more per-mod compiled +// artifact to produce. +type MergeCompiler interface { + // ValidateSource parses/validates sourceFilePath (the retained, + // not-yet-merged source archive) without compiling anything - called at + // ingest time (download/import) so a malformed archive fails loud + // immediately rather than at the next merge. + ValidateSource(sourceFilePath string) error + + // MergeCompile applies every entry in sources, in order (profile load + // order), against basePakPath's tables, and writes the merged result to + // outputPakPath. Returns non-fatal warnings (e.g. same-path asset + // collisions - last-applied wins) alongside a nil error; a nil error + // with warnings is still a fully-written, deployable pak. + MergeCompile(ctx context.Context, basePakPath string, sources []MergeSource, outputPakPath string) (warnings []string, err error) +} + +// MergeSource identifies one mod's contribution to a merge, in the order it +// must be applied (profile load order). +type MergeSource struct { + ModRef string // "sourceID:modID" - identity used in collision warnings + ExmodzPath string // the retained source archive to read +} +``` + +`source.go` already imports `"context"` (used by `ModSource`'s own methods) — no new import needed. + +- [ ] **Step 4: Implement `MergeCompile` and `ValidateSource` in the icarus package** + +Create `internal/source/icarus/merge.go`: + +```go +package icarus + +import ( + "context" + "fmt" + "os" + + "github.com/DonovanMods/linux-mod-manager/internal/source" + "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" +) + +// MergeSource is a type alias (not a distinct type) for source.MergeSource +// (Step 3 above). internal/core must NOT import this icarus package +// directly (established #136/#196 precedent - see +// service_icarus_compile_test.go's fakeCompilerSource doc comment), so it +// can only ever construct/consume source.MergeSource values - aliasing it +// here, rather than defining a second, structurally-similar type, is what +// lets *Icarus's MergeCompile method (Step 6) satisfy source.MergeCompiler +// at all: Go interface satisfaction requires identical types, and a type +// alias IS the same type, not a look-alike. +type MergeSource = source.MergeSource + +// ValidateSource parses exmodzPath without compiling anything - the +// ingest-time check (#197 design: "install still parses/validates the +// .exmodz early"). A malformed archive fails loud immediately, at +// download/import time, rather than at the next merge (which may not run +// until a later mutation). +func ValidateSource(exmodzPath string) error { + data, err := os.ReadFile(exmodzPath) + if err != nil { + return fmt.Errorf("icarus: reading %s: %w", exmodzPath, err) + } + if _, err := ParseExmodz(data); err != nil { + return fmt.Errorf("icarus: validating %s: %w", exmodzPath, err) + } + return nil +} + +// MergeCompile applies every source's .EXMOD row upserts, IN ORDER, against +// the same evolving base tables - a merge is just Compile with N diffs +// instead of 1. Table conflicts compose at the FIELD level for free: +// ApplyRowPatch always shallow-merges an item's fields into whatever the +// target row currently holds, so feeding mod A's patched bytes back in as +// the "base" for mod B's row (instead of re-reading the pristine base table +// each time) is the entire merge algorithm - two mods patching DIFFERENT +// fields of the same row, or entirely different rows of the same table, +// both survive; only a genuine same-row-same-field write is last-wins (an +// ordinary, expected upsert outcome, not something to warn about). Bundled +// ASSET files cannot compose this way - a same-path asset collision is +// necessarily last-wins, so it is reported as a warning instead. +// +// ctx is accepted only to satisfy source.MergeCompiler and is never read - +// every step here is local file I/O over small files (mirrors Compile's own +// doc comment, internal/source/icarus/compile.go:23-25). +// +// A non-nil error always means outputPakPath does not exist (or does not +// contain a fully-written pak) - see the removal defer below, mirroring +// Compile's own fail-clean contract. +func MergeCompile(ctx context.Context, basePakPath string, sources []MergeSource, outputPakPath string) (warnings []string, err error) { + base, err := unrealpak.Open(basePakPath) + if err != nil { + return nil, fmt.Errorf("icarus: opening base pak %s: %w", basePakPath, err) + } + defer base.Close() //nolint:errcheck + + tableState := make(map[string][]byte) // mountPath -> current (possibly already patched) JSON bytes + assets := make(map[string][]byte) // final asset path -> data (last source wins) + assetOwner := make(map[string]string) // asset path -> ModRef that last set it + + for _, src := range sources { + exmodzData, rerr := os.ReadFile(src.ExmodzPath) + if rerr != nil { + return warnings, fmt.Errorf("icarus: reading %s: %w", src.ExmodzPath, rerr) + } + bundle, perr := ParseExmodz(exmodzData) + if perr != nil { + return warnings, fmt.Errorf("icarus: %s: %w", src.ExmodzPath, perr) + } + + for _, row := range bundle.Diff.Rows { + if row.CurrentFile == endOfModSentinel { + continue + } + if len(row.FileItems) == 0 { + return warnings, fmt.Errorf("icarus: %s: row has no File_Items to apply (malformed .EXMOD manifest)", row.CurrentFile) + } + mountPath, merr := resolveCurrentFile(base, row.CurrentFile) + if merr != nil { + return warnings, merr + } + current, seen := tableState[mountPath] + if !seen { + current, merr = base.ReadFile(mountPath) + if merr != nil { + return warnings, fmt.Errorf("icarus: reading base data table %s: %w", mountPath, merr) + } + } + patched, perr2 := ApplyRowPatch(current, row) + if perr2 != nil { + return warnings, perr2 + } + tableState[mountPath] = patched + } + + for assetPath, data := range bundle.Assets { + safePath, serr := sanitizeAssetPath(assetPath) + if serr != nil { + return warnings, serr + } + if owner, exists := assetOwner[safePath]; exists && owner != src.ModRef { + warnings = append(warnings, fmt.Sprintf( + "asset %q is bundled by both %s and %s - %s wins (last-applied, per profile load order)", + safePath, owner, src.ModRef, src.ModRef)) + } + assets[safePath] = data + assetOwner[safePath] = src.ModRef + } + } + + out, cerr := unrealpak.Create(outputPakPath, unrealpak.WithMountPoint(icarusContentMountPoint)) + if cerr != nil { + return warnings, fmt.Errorf("icarus: creating %s: %w", outputPakPath, cerr) + } + defer func() { + if err == nil { + return + } + _ = out.Close() //nolint:errcheck + if rmErr := os.Remove(outputPakPath); rmErr != nil && !os.IsNotExist(rmErr) { + err = fmt.Errorf("%w (additionally, removing partial output %s failed: %v)", err, outputPakPath, rmErr) + } + }() + + for mountPath, data := range tableState { + tablePath := icarusDataTablePrefix + mountPath + if err = out.AddFile(tablePath, data); err != nil { + return warnings, fmt.Errorf("icarus: writing merged %s: %w", tablePath, err) + } + } + for assetPath, data := range assets { + if err = out.AddFile(assetPath, data); err != nil { + return warnings, fmt.Errorf("icarus: writing bundled asset %s: %w", assetPath, err) + } + } + + if err = out.Close(); err != nil { + return warnings, fmt.Errorf("icarus: finalizing %s: %w", outputPakPath, err) + } + return warnings, nil +} +``` + +Note: `ctx` is accepted but unused in the body — `go vet`/the linter will not flag an unused PARAMETER (only unused local variables/imports), so this compiles clean; this exactly mirrors `Compile`'s own sibling situation is avoided (Compile has no ctx at all) but matches how other `ctx`-accepting-but-unused methods already exist elsewhere in this codebase's source implementations. + +Map iteration order for `tableState`/`assets` in the write-out loops is non-deterministic across runs, but `unrealpak.Writer.Close` already sorts all entries by path before serializing (existing behavior, unchanged) — the FINAL pak's byte layout is deterministic regardless of insertion order, so this needs no extra sorting here. + +- [ ] **Step 5: Run the tests, confirm they pass** + +Run: `go test ./internal/source/icarus/... -run 'TestMergeCompile|TestValidateSource' -v` +Expected: all 8 PASS. Extraction-verified: this is the exact code (module path adjusted for the `source.MergeSource` alias, added after the initial extraction pass caught the type-identity issue — see below) proven against a scratch copy of `develop` tip `541b485` with all 6 `MergeCompile` scenarios and both `ValidateSource` cases green. + +**Extraction-verification note:** the FIRST draft of this task defined `MergeSource` as a plain struct local to the `icarus` package (not an alias of `source.MergeSource`). That draft's 6 `MergeCompile` tests still passed — the merge algorithm itself was correct — but a second look while wiring `*Icarus` to the `MergeCompiler` interface (Step 6) surfaced that two structurally-identical-but-distinct Go types never satisfy the same interface: `*Icarus` would NOT have implemented `source.MergeCompiler`. This is fixed by making `icarus.MergeSource` a genuine type alias (`type MergeSource = source.MergeSource`) rather than a second definition, so both packages refer to the exact same type. No change to the merge algorithm itself was needed — this defect was in the type PLUMBING around the verified logic, not the logic. + +- [ ] **Step 6: Run the pre-existing package suite, confirm `Compile`'s own tests still pass** + +Run: `go test ./internal/source/icarus/... -v` +Expected: all PASS (`Compile`, `ApplyRowPatch`, `ParseExmod`, `ParseExmodz` tests all unaffected — nothing about them changed). + +- [ ] **Step 7: Wire `*icarus.Icarus` to implement `MergeCompiler`** + +Find `*icarus.Icarus`'s current `Compile` method (search `internal/source/icarus/icarus.go` for `func (i *Icarus) Compile`). Replace it with: + +```go +func (i *Icarus) ValidateSource(sourceFilePath string) error { + return ValidateSource(sourceFilePath) +} + +func (i *Icarus) MergeCompile(ctx context.Context, basePakPath string, sources []MergeSource, outputPakPath string) ([]string, error) { + return MergeCompile(ctx, basePakPath, sources, outputPakPath) +} +``` + +Find the existing `var _ source.Compiler = (*Icarus)(nil)` (or equivalent) interface-assertion line in `icarus.go` or `icarus_test.go` and change it to `var _ source.MergeCompiler = (*Icarus)(nil)`. + +- [ ] **Step 8: Run the full repo build and test suite, confirm the ONLY breakage is in `internal/core`** + +Run: `go build ./... 2>&1 | tail -60` +Expected: `internal/source/...` builds clean; `internal/core/...` fails to build (`fakeCompilerSource does not implement source.MergeCompiler`, `src.(source.Compiler)` type assertion errors, `compiler.Compile` undefined) — this is the expected, BY-DESIGN breakage Tasks 2-4 fix. Note the exact list of broken files (`internal/core/service.go`, `internal/core/importer.go`, `internal/core/updater.go`, and their test files) for Task 2. + +Run: `go test ./internal/source/... -v 2>&1 | tail -40` +Expected: all green — this package's own suite is fully self-contained and must pass before moving on, independent of `internal/core`'s state. + +- [ ] **Step 9: Commit** + +```bash +git add internal/source/source.go internal/source/icarus/merge.go internal/source/icarus/merge_test.go internal/source/icarus/icarus.go internal/source/icarus/icarus_test.go +git commit -m "feat: MergeCompiler interface + Icarus merge engine (#197)" +``` + +### Task 2: Ingest simplification — download path (`DownloadModToCache`) + +**Files:** + +- Modify: `internal/core/service.go:543-574` (the `DeployCompile` branch inside `DownloadModToCache`) +- Modify: `internal/core/service.go:180-199` (`compilerSourceForGame` → `mergeCompilerSourceForGame`) +- Test: `internal/core/service_icarus_compile_test.go` (update `fakeCompilerSource` to implement `MergeCompiler`; existing per-mod-pak assertions replaced with validate-and-retain assertions) +- Test: `internal/core/service_compile_fingerprint_test.go` (rewritten — see Step 5) + +**Interfaces:** + +- Consumes: `cache.RetainedSourceName(fileID string) string` (#196, unchanged); `commitStagedCacheWithMarker(cachePath, stagePath, fileID string, members []string) error` (#196, unchanged — now always called with `members: nil`); `source.MergeCompiler` (Task 1). +- Produces: after this task, a `DeployCompile` game's per-mod cache entry for an `.exmodz` file contains ONLY the reserved retained-source file — `gameCache.ListFiles(...)` returns an EMPTY slice for it. `DownloadModResult.FilesExtracted` is `0` for this branch (there is nothing to deploy from this mod's own entry — the merged pak, deployed separately in Task 6/7, is what actually reaches the game directory). + +Read `internal/core/service.go:472-596` first (the whole `DownloadModToCache` function) for full context before editing — the snippet below is a targeted diff, not the whole function. + +- [ ] **Step 1: Write the failing test** + +Open `internal/core/service_icarus_compile_test.go`. Replace `fakeCompilerSource`'s `Compile` method: + +```go +func (s *fakeCompilerSource) Compile(ctx context.Context, basePakPath, sourceFilePath, outputPath string) error { + s.compileCalls++ + data, err := os.ReadFile(sourceFilePath) + if err != nil { + return err + } + return os.WriteFile(outputPath, data, 0o644) +} +``` + +with: + +```go +func (s *fakeCompilerSource) ValidateSource(sourceFilePath string) error { + s.validateCalls++ + if _, err := os.Stat(sourceFilePath); err != nil { + return err + } + return nil +} + +func (s *fakeCompilerSource) MergeCompile(ctx context.Context, basePakPath string, sources []source.MergeSource, outputPath string) ([]string, error) { + s.compileCalls++ + // Concatenate every source's bytes - enough for tests to distinguish + // "which sources were actually merged" without needing a real base pak + // table to patch. + var out []byte + for _, src := range sources { + data, err := os.ReadFile(src.ExmodzPath) + if err != nil { + return nil, err + } + out = append(out, data...) + } + return nil, os.WriteFile(outputPath, out, 0o644) +} +``` + +Add `validateCalls int` to the `fakeCompilerSource` struct. No new import is needed — `source.MergeSource` uses the SAME `internal/source` import this file already has for `source.ModSource`/`source.DownloadableFile`-adjacent types (confirm with `grep -n '"github.com/DonovanMods/linux-mod-manager/internal/source"' internal/core/service_icarus_compile_test.go`; do NOT import `internal/source/icarus` into `internal/core` — that import boundary is deliberate, see Step 4's doc comment above). Change the interface assertions: + +```go +var ( + _ source.ModSource = (*fakeCompilerSource)(nil) + _ source.MergeCompiler = (*fakeCompilerSource)(nil) +) +``` + +Replace `TestDownloadMod_DeployCompile_InvokesCompiler` with: + +```go +func TestDownloadMod_DeployCompile_ValidatesAndRetainsNoPerModPak(t *testing.T) { + dlSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("fake-exmodz-bytes")) + })) + defer dlSrv.Close() + + installDir := t.TempDir() + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + writeFakeBasePak(t, basePak) + + cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} + svc, err := core.NewService(cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + src := &fakeCompilerSource{downloadURL: dlSrv.URL} + svc.RegisterSource(src) + + game := &domain.Game{ID: "icarus", InstallPath: installDir, ModPath: t.TempDir(), DeployMode: domain.DeployCompile} + require.NoError(t, svc.AddGame(game)) + + mod := &domain.Mod{ID: "bear-mount", SourceID: "fake-compiler", GameID: "icarus", Version: "3.3"} + file := &domain.DownloadableFile{ID: "exmodz", FileName: "Bear_Mount.exmodz"} + + result, err := svc.DownloadMod(context.Background(), "fake-compiler", game, mod, file, nil) + require.NoError(t, err) + require.Equal(t, 1, src.validateCalls, "ingest must validate the .exmodz") + require.Equal(t, 0, src.compileCalls, "ingest must NOT compile a per-mod pak (#197: merged-only)") + require.Equal(t, 0, result.FilesExtracted, "a per-mod exmodz cache entry has no deployment members under the merged-only model") + + gameCache := svc.GetGameCache(game) + files, err := gameCache.ListFiles(game.ID, mod.SourceID, mod.ID, mod.Version) + require.NoError(t, err) + require.Empty(t, files, "ListFiles must report zero deployment members - the retained source is reserved, not a member") + + retainedPath := gameCache.GetFilePath(game.ID, mod.SourceID, mod.ID, mod.Version, cache.RetainedSourceName(file.ID)) + data, err := os.ReadFile(retainedPath) + require.NoError(t, err) + require.Equal(t, "fake-exmodz-bytes", string(data), "the original .exmodz bytes must still be retained") +} + +// TestDownloadMod_DeployCompile_MalformedExmodz_FailsLoudAtIngest proves +// validation happens at ingest time, not deferred to the next merge. +func TestDownloadMod_DeployCompile_MalformedExmodz_FailsLoudAtIngest(t *testing.T) { + dlSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("not-a-valid-exmodz")) + })) + defer dlSrv.Close() + + installDir := t.TempDir() + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + writeFakeBasePak(t, basePak) + + cfg := core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()} + svc, err := core.NewService(cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + src := &failingValidateCompilerSource{fakeCompilerSource: &fakeCompilerSource{downloadURL: dlSrv.URL}} + svc.RegisterSource(src) + + game := &domain.Game{ID: "icarus", InstallPath: installDir, ModPath: t.TempDir(), DeployMode: domain.DeployCompile} + require.NoError(t, svc.AddGame(game)) + + mod := &domain.Mod{ID: "bad-mount", SourceID: "fake-compiler", GameID: "icarus", Version: "1.0"} + file := &domain.DownloadableFile{ID: "exmodz", FileName: "Bad_Mount.exmodz"} + + _, err = svc.DownloadMod(context.Background(), "fake-compiler", game, mod, file, nil) + require.Error(t, err) + + gameCache := svc.GetGameCache(game) + require.False(t, gameCache.Exists(game.ID, mod.SourceID, mod.ID, mod.Version), "a validation failure must leave no cache entry") +} + +// failingValidateCompilerSource wraps fakeCompilerSource and always fails +// ValidateSource - simulates a corrupt/malformed downloaded .exmodz. +type failingValidateCompilerSource struct { + *fakeCompilerSource +} + +func (s *failingValidateCompilerSource) ValidateSource(sourceFilePath string) error { + return fmt.Errorf("boom: not a valid .EXMODZ") +} +``` + +Add `"fmt"` and `"github.com/DonovanMods/linux-mod-manager/internal/storage/cache"` to this test file's imports if not already present (they are — `cache` is used elsewhere in this package's test files; confirm with `grep -n '"github.com/DonovanMods/linux-mod-manager/internal/storage/cache"' internal/core/service_icarus_compile_test.go` before adding a duplicate). + +- [ ] **Step 2: Run the tests, confirm they fail to compile** + +Run: `go test ./internal/core/... -run 'TestDownloadMod_DeployCompile' -v` +Expected: build failure (`fakeCompilerSource` doesn't implement `source.MergeCompiler`, `validateCalls` undefined) — this IS the expected RED state; Step 3 (below) makes it compile. + +- [ ] **Step 3: Replace the `DeployCompile` branch** + +In `internal/core/service.go`, replace lines 543-574 (quoted above in "Files") with: + +```go + if game.DeployMode == domain.DeployCompile && isExmodzFile(safeFileName) { + mc, ok := src.(source.MergeCompiler) + if !ok { + return nil, fmt.Errorf("source %q: game %q requires DeployCompile but source does not implement MergeCompiler", src.ID(), game.ID) + } + if err := mc.ValidateSource(archivePath); err != nil { + return nil, fmt.Errorf("validating %s: %w", safeFileName, err) + } + // Unlike copyFileStreaming (which mkdirs its destination itself), + // the retained-source write below needs stagePath to exist first. + if err := os.MkdirAll(stagePath, 0755); err != nil { + return nil, fmt.Errorf("preparing staging: %w", err) + } + retainedPath := filepath.Join(stagePath, cache.RetainedSourceName(file.ID)) + if err := copyFileStreaming(archivePath, retainedPath); err != nil { + return nil, fmt.Errorf("retaining %s: %w", safeFileName, err) + } + // members is nil (#197): this cache entry's ONLY content is the + // reserved retained source - there is no per-mod deployment + // artifact anymore. The merged pak (a separate, profile-level + // cache entry - internal/core/merged_pak.go) is what actually + // deploys. + if err := commitStagedCacheWithMarker(cachePath, stagePath, file.ID, nil); err != nil { + return nil, err + } + return &DownloadModResult{FilesExtracted: 0, Checksum: downloadResult.Checksum}, nil + } +``` + +Do NOT delete `resolveBasePak`, `compiledFileName`, `basePakIndexHash`, or `stageCompileFingerprint` yet — `resolveBasePak` and `basePakIndexHash` are still used by Task 5/6 (the merged pak's own fingerprint needs the live base pak's `IndexHash`); `compiledFileName` and `stageCompileFingerprint` become genuinely dead here and are removed in Task 4 once every caller is confirmed gone (removing them now, before Task 3 also stops calling them, would break the build for the importer branch mid-task). + +- [ ] **Step 4: Rename `compilerSourceForGame` to `mergeCompilerSourceForGame`** + +`internal/core/service.go:180-199`. This function's body is otherwise unchanged except its return type and the type assertion inside the loop: + +```go +func (s *Service) mergeCompilerSourceForGame(gameID string) (source.MergeCompiler, error) { + srcs, err := s.SourcesForGame(gameID) + if err != nil { + return nil, err + } + var compilers []source.MergeCompiler + for _, src := range srcs { + if c, ok := src.(source.MergeCompiler); ok { + compilers = append(compilers, c) + } + } + switch len(compilers) { + case 0: + return nil, fmt.Errorf("game %q requires DeployCompile but has no merge-compiler-capable source configured (map a source implementing source.MergeCompiler in the game's sources)", gameID) + case 1: + return compilers[0], nil + default: + return nil, fmt.Errorf("game %q has multiple merge-compiler-capable sources configured; ambiguous compile source", gameID) + } +} +``` + +Run `grep -rn "compilerSourceForGame" internal/core/*.go` and update every call site to the new name (Task 3's `Importer.Import` is the only other caller — updated there; Task 5/6's `syncMergedPak` is a NEW caller, using the new name from the start). + +- [ ] **Step 5: Run the tests, confirm they pass** + +Run: `go test ./internal/core/... -run 'TestDownloadMod_DeployCompile' -v` +Expected: both new tests PASS. This will also break every OTHER `service_icarus_compile_test.go`/`service_compile_fingerprint_test.go` test still asserting on per-mod pak output (`TestDownloadMod_DeployCompile_RoutesPerFile`, `TestDownloadMod_DeployCompile_MixedFileMod`, `TestDownloadMod_DeployCompile_RecordsBaseIndexHashAndRetainedSource`) — expected; Step 6 removes/rewrites them. + +- [ ] **Step 6: Remove now-obsolete per-mod-pak tests** + +Delete `TestDownloadMod_DeployCompile_RoutesPerFile` and `TestDownloadMod_DeployCompile_MixedFileMod` from `service_icarus_compile_test.go` (their premise — a per-mod compiled pak with a specific filename — no longer exists; Task 1's `TestMergeCompile_*` tests and this task's new tests cover the equivalent ground for the merged model). Delete `internal/core/service_compile_fingerprint_test.go` entirely (both its tests assert on `cache.BaseIndexHashes`/retained-source-plus-marker shape that Task 4 removes) — a replacement fingerprint test lives in Task 5. + +- [ ] **Step 7: Run the full core suite, confirm the remaining failures are ONLY in files Task 3/4 own** + +Run: `go test ./internal/core/... 2>&1 | tail -80` +Expected: failures confined to `importer.go`'s own `DeployCompile` branch (still calling the old `Compile`-based flow) and `updater.go`'s `CheckBaseStaleness`/`ApplyRecompile` (still referencing the removed `MarkBaseIndexHash`-based per-mod fingerprint) — both fixed in Tasks 3-4/6/9. Confirm nothing in `internal/tui` or `cmd/lmm` is affected yet (their turn is Tasks 10/12). + +- [ ] **Step 8: Commit** + +```bash +git add internal/core/service.go internal/core/service_icarus_compile_test.go +git rm internal/core/service_compile_fingerprint_test.go +git commit -m "feat: download path ingests .exmodz as validate+retain, no per-mod pak (#197)" +``` + +### Task 3: Ingest simplification — import path (`Importer.Import`) + +**Files:** + +- Modify: `internal/core/importer.go:40-48` (`resolveCompiler` field type) +- Modify: `internal/core/importer.go:65` (`Service.NewImporter`) +- Modify: `internal/core/importer.go:110-184` (the `DeployCompile` branch inside `Import`) +- Test: `internal/core/service_import_compile_test.go` (rewrite the compile-branch fakes/assertions) + +**Interfaces:** + +- Consumes: `source.MergeCompiler` (Task 1); `mergeCompilerSourceForGame` (Task 2, Step 4); `cache.RetainedSourceName` (#196). +- Produces: after this task, an imported `.exmodz`'s cache entry contains ONLY the reserved retained-source file — `result.FilesExtracted` is `0` for this branch, matching Task 2's download-path behavior exactly. + +- [ ] **Step 1: Write the failing test** + +Open `internal/core/service_import_compile_test.go`. Update `fakeCompilerSource` there the same way as Task 2 Step 1 (this file has its own copy per the existing `#173`-era test structure — confirm with `grep -n "type fakeCompilerSource" internal/core/*.go`; if Task 2 already made it shared/exported across test files in this package, skip re-declaring and just import the one type). Replace `TestImportMod_DeployCompile_ExmodzCompiles` and `TestImportMod_DeployCompile_RecordsBaseIndexHashAndRetainedSource` with: + +```go +func TestImportMod_DeployCompile_ValidatesAndRetainsNoPerModPak(t *testing.T) { + svc, src, game := newImportCompileTestGame(t) + + tempDir := t.TempDir() + archivePath := filepath.Join(tempDir, "Bear_Mount.exmodz") + require.NoError(t, os.WriteFile(archivePath, []byte("fake-exmodz-bytes"), 0o644)) + + importer := svc.NewImporter(game) + result, err := importer.Import(context.Background(), archivePath, game, core.ImportOptions{}) + require.NoError(t, err) + require.Equal(t, 1, src.validateCalls) + require.Equal(t, 0, src.compileCalls, "import must NOT compile a per-mod pak (#197: merged-only)") + require.Equal(t, 0, result.FilesExtracted) + + gameCache := svc.GetGameCache(game) + files, err := gameCache.ListFiles(game.ID, result.Mod.SourceID, result.Mod.ID, result.Mod.Version) + require.NoError(t, err) + require.Empty(t, files) + + // Import has no real DownloadableFile.ID (see the field's own doc + // comment) - it keys the retained source by the ARCHIVE'S OWN filename + // instead, exactly as the #196-era destName-keying did. + retainedPath := gameCache.GetFilePath(game.ID, result.Mod.SourceID, result.Mod.ID, result.Mod.Version, cache.RetainedSourceName("Bear_Mount.exmodz")) + data, err := os.ReadFile(retainedPath) + require.NoError(t, err) + require.Equal(t, "fake-exmodz-bytes", string(data)) +} + +func TestImportMod_DeployCompile_MalformedExmodz_FailsLoud(t *testing.T) { + svc, src, game := newImportCompileTestGame(t) + _ = src // validation failure is injected by wrapping, not by this fake + + failing := &failingValidateCompilerSource{fakeCompilerSource: &fakeCompilerSource{}} + // Re-register under the same source ID so the importer resolves the + // failing wrapper instead of the passing fake newImportCompileTestGame + // already registered. + svc.RegisterSource(failing) + + tempDir := t.TempDir() + archivePath := filepath.Join(tempDir, "Bad_Mount.exmodz") + require.NoError(t, os.WriteFile(archivePath, []byte("not-a-valid-exmodz"), 0o644)) + + importer := svc.NewImporter(game) + _, err := importer.Import(context.Background(), archivePath, game, core.ImportOptions{}) + require.Error(t, err) +} +``` + +`newImportCompileTestGame` already exists in this file (from #173/#196) and registers `fakeCompilerSource{}` under source ID `"fake-compiler"` with `game.SourceIDs = {"fake-compiler": ...}` — since `svc.RegisterSource` overwrites by ID (confirm: `grep -n "func.*RegisterSource" internal/core/service.go` shows it stores into a map keyed by `src.ID()`, and `failingValidateCompilerSource` embeds `*fakeCompilerSource{}` whose `ID()` returns `"fake-compiler"` too), the second `RegisterSource` call replaces the passing fake for that source ID within this one test — no `game.SourceIDs` change needed. + +- [ ] **Step 2: Run the tests, confirm they fail to compile** + +Run: `go test ./internal/core/... -run 'TestImportMod_DeployCompile' -v` +Expected: build failure (`validateCalls` undefined on this file's copy of `fakeCompilerSource` until it's updated identically to Task 2 Step 1). + +- [ ] **Step 3: Replace the `DeployCompile` branch** + +`internal/core/importer.go:40-48` — change the field: + +```go + // resolveMergeCompiler resolves the MergeCompiler-capable source mapped + // to a DeployCompile game's registry entry (#197), consulted only when + // importing a ".exmodz" archive for such a game — Import has no + // per-archive source pinned the way DownloadModToCache does, so it must + // look up the game's configured sources instead. nil when the Importer + // was built via the standalone NewImporter (no Service context): + // importing an .exmodz through such an Importer fails loud rather than + // silently caching an unvalidated archive. + resolveMergeCompiler func(gameID string) (source.MergeCompiler, error) +``` + +`internal/core/importer.go:65` — update the assignment: + +```go + imp.resolveMergeCompiler = s.mergeCompilerSourceForGame +``` + +`internal/core/importer.go:110-184` — replace the entire `if game.DeployMode == domain.DeployCompile && isExmodzFile(filename) {` block body with: + +```go + if game.DeployMode == domain.DeployCompile && isExmodzFile(filename) { + // Validate mode (#197): Import has no real source file ID the way a + // download does (DownloadableFile.ID is resolved later, outside + // Import, only when --id was given), so the retained source is + // keyed by the archive's own filename instead - stable across + // re-imports of the same name, and the ONLY identity Import ever + // has for this content. + if i.resolveMergeCompiler == nil { + return nil, fmt.Errorf("game %q requires DeployCompile to import %q, but this Importer was constructed without service context (via core.NewImporter, not Service.NewImporter) and has no compiler resolver to consult - import via the service-backed importer instead", game.ID, filename) + } + mc, err := i.resolveMergeCompiler(game.ID) + if err != nil { + return nil, err + } + if err := mc.ValidateSource(archivePath); err != nil { + return nil, fmt.Errorf("validating %s: %w", filename, err) + } + + modName = strings.TrimSuffix(filename, filepath.Ext(filename)) + if version != "" && version != "unknown" { + if idx := strings.LastIndex(modName, version); idx > 0 { + modName = strings.TrimRight(modName[:idx], "-_ ") + } + } + + cacheMod := &domain.Mod{ID: modID, SourceID: sourceID, Version: version, GameID: game.ID} + cachePath, stagePath, err := prepareUnseededStaging(i.cache, game, cacheMod) + if err != nil { + return nil, err + } + defer os.RemoveAll(stagePath) //nolint:errcheck + + if err := os.MkdirAll(stagePath, 0755); err != nil { + return nil, fmt.Errorf("preparing cache staging: %w", err) + } + retainedPath := filepath.Join(stagePath, cache.RetainedSourceName(filename)) + if err := copyFileStreaming(archivePath, retainedPath); err != nil { + return nil, fmt.Errorf("retaining %s: %w", filename, err) + } + if err := commitStagedCache(cachePath, stagePath); err != nil { + return nil, err + } + fileCount = 0 + } else if game.DeployMode == domain.DeployCopy { +``` + +(The `} else if game.DeployMode == domain.DeployCopy {` on the last line is the pre-existing next branch — reattach it exactly as it already reads at the current line 185; do not duplicate it.) Add `"github.com/DonovanMods/linux-mod-manager/internal/storage/cache"` to `importer.go`'s import block if not already present (`grep -n '"github.com/DonovanMods/linux-mod-manager/internal/storage/cache"' internal/core/importer.go` — it likely already is, since `i.cache *cache.Cache` is an existing field type). + +- [ ] **Step 4: Run the tests, confirm they pass** + +Run: `go test ./internal/core/... -run 'TestImportMod_DeployCompile' -v` +Expected: both new tests PASS. + +- [ ] **Step 5: Remove now-obsolete per-mod-pak import tests** + +Delete `TestImportMod_DeployCompile_RoutesPerFile`, `TestImportMod_DeployCompile_CompileFailureLeavesNoPartialArtifact`, `TestImportMod_DeployCompile_ReimportSurvivesStagingFailure` from `service_import_compile_test.go` — each asserts on the removed per-mod-pak-compile shape (destName-as-`_P.pak`, a `raceCompilerSource`/`failingCompilerSource` simulating a COMPILE failure, which no longer exists as a per-mod step here — validation failure is now the only failure mode Import's own branch can produce, already covered by Step 1's new test). Keep `TestImportMod_DeployCompile_ZipPassthroughUnaffected`, `TestImportMod_DeployCompile_NoCompilerSourceFailsLoud` (rename its fake-lookup assertion target from `resolveCompiler`/`compilerSourceForGame` wording to `resolveMergeCompiler`/`mergeCompilerSourceForGame` if the test's own comment or error-string assertion names it), `TestImportMod_DeployCompile_MissingBasePakFailsLoud` — **wait**, re-read this last one: Import's `DeployCompile` branch after this task NO LONGER calls `resolveBasePak` at all (validation doesn't need a base pak — only the eventual MERGE does). Delete `TestImportMod_DeployCompile_MissingBasePakFailsLoud` too; there is nothing base-pak-related left to fail on at import time. `TestImportMod_DeployCompile_StandaloneImporterFailsLoud` stays, renaming its assertion string check (`"without service context"`/`"core.NewImporter"`) — unaffected wording, still correct. + +- [ ] **Step 6: Run the full core suite, confirm remaining failures are confined to `updater.go`** + +Run: `go test ./internal/core/... 2>&1 | tail -80` +Expected: `internal/core/service.go` and `internal/core/importer.go` compile and their own tests pass; remaining failures are in `updater.go`'s `CheckBaseStaleness`/`ApplyRecompile` (still referencing removed per-mod machinery) — fixed in Task 4/6/9. + +- [ ] **Step 7: Commit** + +```bash +git add internal/core/importer.go internal/core/service_import_compile_test.go +git commit -m "feat: import path ingests .exmodz as validate+retain, no per-mod pak (#197)" +``` + +### Task 4: Remove dead #196 per-mod compile machinery + +**Files:** + +- Modify: `internal/storage/cache/cache.go:253-312` (remove `baseIndexHashPrefix`/`MarkBaseIndexHash`/`BaseIndexHashes`) +- Modify: `internal/storage/cache/cache_test.go` (remove their tests) +- Modify: `internal/core/service.go:1008-1054` (remove `compiledFileName`, `stageCompileFingerprint`; KEEP `resolveBasePak` and `basePakIndexHash` — Task 5 reuses both) + +**Interfaces:** + +- Consumes: nothing new. +- Produces: nothing new — this is pure removal. `resolveBasePak(game *domain.Game) (string, error)` and `basePakIndexHash(basePakPath string) (string, error)` remain exactly as-is (Task 5's `syncMergedPak` calls both). + +- [ ] **Step 1: Confirm nothing outside this task's own scope still calls the functions being removed** + +Run: `grep -rn "MarkBaseIndexHash\|BaseIndexHashes\|compiledFileName\|stageCompileFingerprint" --include='*.go' .` +Expected, after Tasks 2/3 landed: matches ONLY inside `internal/storage/cache/cache.go`/`cache_test.go` (definitions) and `internal/core/service.go` (definitions) — zero call sites left in `internal/core/updater.go` (Task 6/9 will have already stopped calling them if done in order; if this task runs before Task 6/9 in a different execution order, `updater.go`'s `CheckBaseStaleness`/`ApplyRecompile` will still reference `BaseIndexHashes` — in that case, do Task 6 first, or accept that `go build` fails here until Task 6 lands, which is fine since this whole plan's tasks are meant to run in the numbered order). + +- [ ] **Step 2: Remove `baseIndexHashPrefix`/`MarkBaseIndexHash`/`BaseIndexHashes` from `cache.go`** + +Delete lines 253-312 (quoted in full in this task's "Files" section context above) verbatim — from the `// baseIndexHashPrefix names...` comment through the `BaseIndexHashes` function's closing `}`. The following `retainedSourcePrefix`/`RetainedSourceName` block (lines 314-329) is untouched and now becomes the section immediately following whatever preceded line 253 (`HasFileIDs`, unaffected). + +- [ ] **Step 3: Remove their tests from `cache_test.go`** + +Run: `grep -n "^func TestCache_BaseIndexHashes\|^func TestCache_MarkBaseIndexHash" internal/storage/cache/cache_test.go` and delete `TestCache_BaseIndexHashes_RoundTrip`, `TestCache_BaseIndexHashes_ExcludedFromContentEnumerators`, `TestCache_MarkBaseIndexHash_UnverifiableIDs` in full. Leave `TestCache_RetainedSourceName_IsReservedAndExcludedFromContent` and `TestCache_RetainedSourceName_UniquePerFileID` — unaffected. + +- [ ] **Step 4: Remove `compiledFileName`/`stageCompileFingerprint` from `service.go`** + +`internal/core/service.go:1008-1017` (`compiledFileName`) and `:1036-1054` (`stageCompileFingerprint`) — delete both functions in full (their doc comments too). `basePakIndexHash` (currently between them) stays. After deletion, `resolveBasePak` and `basePakIndexHash` should be adjacent (or separated only by whatever other unrelated function already sat between `resolveBasePak` and `compiledFileName`). + +- [ ] **Step 5: Run the full build** + +Run: `go build ./... 2>&1 | tail -60` +Expected (if run in plan order, after Task 6/9): clean build. If any `unused` warnings appear for imports that only `compiledFileName`/`stageCompileFingerprint` needed, remove them (`go vet`/`gofmt` will not catch unused imports, but `go build` will fail loudly on them — check `internal/core/service.go`'s import block against what's still referenced). + +- [ ] **Step 6: Run the full test suite** + +Run: `go test ./... 2>&1 | tail -60` +Expected: green, assuming Task 5/6/9 have landed first to remove the LAST remaining callers (`updater.go`). + +- [ ] **Step 7: Commit** + +```bash +git add internal/storage/cache/cache.go internal/storage/cache/cache_test.go internal/core/service.go +git commit -m "chore: remove dead #196 per-mod compile fingerprint machinery (#197)" +``` + +**Note on task ordering:** this task is listed 4th for narrative clarity (finish the ingest-side cleanup before moving to the new merge/deploy machinery), but its Step 5/6 depend on Task 6/9 already having stopped calling `BaseIndexHashes`/`ApplyRecompile`. If executing tasks strictly in order, expect `go build` to fail between Task 4 and Task 6/9 landing — that is fine; Task 4's own commit still only touches the files listed above, and the build goes green once Task 9 lands. Subagent-driven execution should sequence Tasks 5→6→7→8→9 before circling back to finish Task 4's Step 5-7, or simply do Task 4 last, after Task 9 — both orderings produce the identical final diff. + +### Task 5: `MergedFingerprint` type + cache marker path + `enabledExmodzSources` + +**Files:** + +- Modify: `internal/domain/mod.go` (add `SourceMerged` constant) +- Modify: `internal/storage/cache/cache.go` (add `MergeFingerprintPath`) +- Test: `internal/storage/cache/cache_test.go` +- Create: `internal/core/merged_pak.go` +- Test: `internal/core/merged_pak_test.go` + +**Interfaces:** + +- Consumes: `cache.ReservedPrefix` (#196, unchanged); `Service.GetInstalledModsInProfileOrder(gameID, profileName string) ([]domain.InstalledMod, error)` (existing, unchanged); `cache.RetainedSourceName(fileID string) string` (#196, unchanged); `md5File(path string) (string, error)` (#196, unchanged); `resolveBasePak`/`basePakIndexHash` (#196, unchanged — kept by Task 4). +- Produces: `domain.SourceMerged = "lmm-merged"`; `cache.MergeFingerprintPath(versionDir string) string`; `core.MergedFingerprint{BaseIndexHash string, Mods []MergedFingerprintEntry}`; `core.MergedFingerprintEntry{SourceID, ModID, Version, Checksum string}`; `core.marshalMergedFingerprint(f MergedFingerprint) ([]byte, error)`; `core.mergedFingerprintsEqual(a, b MergedFingerprint) (bool, error)`; `core.mergedPakModID = "merged-pak"`, `core.mergedPakVersion = "merged"`, `core.mergedPakFileName = "zzz_LMM_Merged_P.pak"`; `Service.enabledExmodzSources(game *domain.Game, profileName string) ([]source.MergeSource, error)`. All consumed by Task 6. + +**Extraction-verified:** `MergedFingerprint`/`marshalMergedFingerprint`/`mergedFingerprintsEqual` below are the EXACT code verified against the scratch copy of `develop` tip `541b485`, with 7 tests proving determinism and that every documented regeneration trigger (base pak change, mod enabled/disabled, load-order swap, version bump) produces an unequal comparison. **One real defect was caught and fixed during verification:** `encoding/json` marshals a nil slice as `null` but an empty slice as `[]` — two different byte sequences for what must count as the same "zero contributing mods" state. Without the normalization in `marshalMergedFingerprint` below, a profile with zero enabled exmodz mods could spuriously flip between "stale"/"not stale" depending on which code path happened to build each side's slice. The fix (normalize `nil` to `[]MergedFingerprintEntry{}` before marshaling) is included below, not a follow-up. + +- [ ] **Step 1: Write the failing tests** + +`internal/domain/mod.go` — no test needed for a bare constant; add it directly in Step 4 below. + +Create `internal/storage/cache/cache_test.go` additions (append to the existing file, do not create a new one): + +```go +func TestCache_MergeFingerprintPath_IsReserved(t *testing.T) { + path := cache.MergeFingerprintPath("/some/version/dir") + if !strings.HasPrefix(filepath.Base(path), cache.ReservedPrefix) { + t.Errorf("MergeFingerprintPath = %q, want a reserved-prefixed basename", path) + } +} + +func TestCache_MergeFingerprintPath_ExcludedFromContent(t *testing.T) { + c := cache.New(t.TempDir()) + require.NoError(t, c.Store("g", "lmm-merged", "merged-pak", "merged", "zzz_LMM_Merged_P.pak", []byte("pak-bytes"))) + versionDir := c.ModPath("g", "lmm-merged", "merged-pak", "merged") + require.NoError(t, os.WriteFile(cache.MergeFingerprintPath(versionDir), []byte(`{"BaseIndexHash":"abc"}`), 0o644)) + + files, err := c.ListFiles("g", "lmm-merged", "merged-pak", "merged") + require.NoError(t, err) + assert.Equal(t, []string{"zzz_LMM_Merged_P.pak"}, files, "the fingerprint marker must never be listed as deployable content") +} +``` + +Create `internal/core/merged_pak_test.go`: + +```go +package core_test + +import "testing" + +// These tests exercise unexported core package internals (marshalMergedFingerprint, +// mergedFingerprintsEqual) and therefore live in package core, not core_test - +// see merged_pak_internal_test.go. +var _ = testing.T{} +``` + +Create `internal/core/merged_pak_internal_test.go` (white-box, `package core` — matches this package's existing precedent of a mixed black-box/white-box test split, e.g. `service_download_local_test.go` is `package core` while most other test files here are `package core_test`): + +```go +package core + +import "testing" + +func TestMergedFingerprint_Deterministic(t *testing.T) { + f := MergedFingerprint{ + BaseIndexHash: "abc123", + Mods: []MergedFingerprintEntry{ + {SourceID: "icarus", ModID: "bear-mount", Version: "1.0", Checksum: "deadbeef"}, + {SourceID: "icarus", ModID: "wolf-mount", Version: "2.0", Checksum: "cafef00d"}, + }, + } + b1, err := marshalMergedFingerprint(f) + if err != nil { + t.Fatalf("marshal 1: %v", err) + } + b2, err := marshalMergedFingerprint(f) + if err != nil { + t.Fatalf("marshal 2: %v", err) + } + if string(b1) != string(b2) { + t.Errorf("marshal not deterministic: %q vs %q", b1, b2) + } +} + +func TestMergedFingerprintsEqual_IdenticalInputs(t *testing.T) { + f := MergedFingerprint{ + BaseIndexHash: "abc123", + Mods: []MergedFingerprintEntry{{SourceID: "icarus", ModID: "bear-mount", Version: "1.0", Checksum: "deadbeef"}}, + } + eq, err := mergedFingerprintsEqual(f, f) + if err != nil { + t.Fatalf("mergedFingerprintsEqual: %v", err) + } + if !eq { + t.Errorf("identical fingerprints compared unequal") + } +} + +func TestMergedFingerprintsEqual_BaseHashChanged(t *testing.T) { + a := MergedFingerprint{BaseIndexHash: "abc123", Mods: []MergedFingerprintEntry{{SourceID: "icarus", ModID: "m1", Version: "1.0", Checksum: "x"}}} + b := a + b.BaseIndexHash = "def456" + eq, err := mergedFingerprintsEqual(a, b) + if err != nil { + t.Fatalf("mergedFingerprintsEqual: %v", err) + } + if eq { + t.Errorf("base pak change (regeneration trigger) must compare unequal") + } +} + +func TestMergedFingerprintsEqual_ModSetChanged(t *testing.T) { + a := MergedFingerprint{BaseIndexHash: "abc", Mods: []MergedFingerprintEntry{{SourceID: "icarus", ModID: "m1", Version: "1.0", Checksum: "x"}}} + b := MergedFingerprint{BaseIndexHash: "abc", Mods: []MergedFingerprintEntry{ + {SourceID: "icarus", ModID: "m1", Version: "1.0", Checksum: "x"}, + {SourceID: "icarus", ModID: "m2", Version: "1.0", Checksum: "y"}, + }} + eq, err := mergedFingerprintsEqual(a, b) + if err != nil { + t.Fatalf("mergedFingerprintsEqual: %v", err) + } + if eq { + t.Errorf("enabling a mod (regeneration trigger) must compare unequal") + } +} + +func TestMergedFingerprintsEqual_LoadOrderChanged(t *testing.T) { + a := MergedFingerprint{BaseIndexHash: "abc", Mods: []MergedFingerprintEntry{ + {SourceID: "icarus", ModID: "m1", Version: "1.0", Checksum: "x"}, + {SourceID: "icarus", ModID: "m2", Version: "1.0", Checksum: "y"}, + }} + b := MergedFingerprint{BaseIndexHash: "abc", Mods: []MergedFingerprintEntry{ + {SourceID: "icarus", ModID: "m2", Version: "1.0", Checksum: "y"}, + {SourceID: "icarus", ModID: "m1", Version: "1.0", Checksum: "x"}, + }} + eq, err := mergedFingerprintsEqual(a, b) + if err != nil { + t.Fatalf("mergedFingerprintsEqual: %v", err) + } + if eq { + t.Errorf("a load-order swap (regeneration trigger) must compare unequal, got equal") + } +} + +func TestMergedFingerprintsEqual_VersionChanged(t *testing.T) { + a := MergedFingerprint{BaseIndexHash: "abc", Mods: []MergedFingerprintEntry{{SourceID: "icarus", ModID: "m1", Version: "1.0", Checksum: "x"}}} + b := MergedFingerprint{BaseIndexHash: "abc", Mods: []MergedFingerprintEntry{{SourceID: "icarus", ModID: "m1", Version: "2.0", Checksum: "x2"}}} + eq, err := mergedFingerprintsEqual(a, b) + if err != nil { + t.Fatalf("mergedFingerprintsEqual: %v", err) + } + if eq { + t.Errorf("a mod version bump (regeneration trigger) must compare unequal") + } +} + +func TestMergedFingerprintsEqual_EmptyModsBothSides(t *testing.T) { + a := MergedFingerprint{BaseIndexHash: "abc", Mods: nil} + b := MergedFingerprint{BaseIndexHash: "abc", Mods: []MergedFingerprintEntry{}} + eq, err := mergedFingerprintsEqual(a, b) + if err != nil { + t.Fatalf("mergedFingerprintsEqual: %v", err) + } + if !eq { + t.Errorf("nil vs empty Mods slice must still compare equal (both marshal to the same JSON array shape)") + } +} +``` + +Delete the placeholder `internal/core/merged_pak_test.go` stub above once `merged_pak_internal_test.go` exists — it was only there to document the black-box/white-box split decision; the REAL black-box tests for `enabledExmodzSources` are added in Step 5 below, in this same (deleted-then-recreated) file. + +- [ ] **Step 2: Run the tests, confirm they fail to compile** + +Run: `go test ./internal/storage/cache/... ./internal/core/... -run 'TestCache_MergeFingerprintPath|TestMergedFingerprint' -v` +Expected: build failure — `undefined: cache.MergeFingerprintPath`, `undefined: MergedFingerprint`, etc. + +- [ ] **Step 3: Add `cache.MergeFingerprintPath`** + +`internal/storage/cache/cache.go`, right after the existing `RetainedSourceName` function (end of that #196 block): + +```go +// mergeFingerprintMarkerName names the single JSON fingerprint marker a +// merged-pak cache entry carries (#197): what base pak and which +// (source, mod, version, exmodz-checksum) tuples, in order, the pak was +// last built from - so a later staleness check can compare without +// re-deriving the merge. Reserved (ReservedPrefix) so ListFiles/Size/deploy +// skip it like every other lmm bookkeeping entry. +const mergeFingerprintMarkerName = ReservedPrefix + "merge-fingerprint" + +// MergeFingerprintPath returns the reserved on-disk path for versionDir's +// merge-fingerprint marker. Pure naming, like RetainedSourceName - callers +// (internal/core, which owns the MergedFingerprint type and its JSON +// encoding) read/write the actual bytes with ordinary file I/O. +func MergeFingerprintPath(versionDir string) string { + return filepath.Join(versionDir, mergeFingerprintMarkerName) +} +``` + +- [ ] **Step 4: Add `domain.SourceMerged`** + +`internal/domain/mod.go`, right after the existing `SourceLocal` constant: + +```go +// SourceMerged is the source ID for the synthetic, profile-scoped "mod" +// that tracks a game's merged compiled pak (#197 - Icarus's cross-mod +// table merge). Follows the SourceLocal precedent: a reserved sentinel +// string, not a real ModSource registration. +const SourceMerged = "lmm-merged" +``` + +- [ ] **Step 5: Implement `MergedFingerprint` and `enabledExmodzSources`** + +Create `internal/core/merged_pak.go`: + +```go +package core + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/source" + "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" +) + +// mergedPakModID/mergedPakVersion/mergedPakFileName identify the merged pak +// as a synthetic, singleton "mod" per (game, profile) - domain.SourceMerged +// is the matching sourceID. This reuses Installer.Install/Uninstall and +// cache.Cache verbatim (#197 design decision 2) rather than a parallel +// deploy/tracking mechanism: zero schema changes, and the SAME +// deployed_files ownership (and #168-class residue risk) as every other +// deployed file. +const ( + mergedPakModID = "merged-pak" + // mergedPakVersion is fixed ("merged", not a real upstream version) - + // there is exactly one merged pak per (game, profile) at any time, and + // every regeneration REPLACES it outright (mirrors #166's directory- + // source "replace, don't overlay" precedent) rather than versioning it. + mergedPakVersion = "merged" + // mergedPakFileName sorts LAST among files UE mounts from a profile's + // mods directory: paks mount in filename-sort order and a later mount + // wins same-path conflicts (this repo's own icarusContentMountPoint doc + // comment, and #197's issue body, both note this) - "zzz" is a + // long-standing UE-modding convention for "load last, highest + // priority", so the merged pak's authoritative combined table state can + // never be silently shadowed by a plain prebuilt .pak mod that happens + // to also carry a table override. "LMM" makes the file greppable as + // lmm-owned; "_P" matches this codebase's existing override-pak suffix + // convention (compiledFileName). + mergedPakFileName = "zzz_LMM_Merged_P.pak" +) + +// MergedFingerprint captures everything a merged pak was built from (#197): +// the base pak's IndexHash plus an ORDERED list of every contributing +// exmodz file's identity and content checksum. Order matters - it's the +// profile's load order, which is also merge-application order - so two +// fingerprints with the same entries in a DIFFERENT order must compare +// unequal (a load-order change is a documented regeneration trigger). +type MergedFingerprint struct { + BaseIndexHash string + Mods []MergedFingerprintEntry +} + +// MergedFingerprintEntry identifies one contributing file within a +// MergedFingerprint. +type MergedFingerprintEntry struct { + SourceID string + ModID string + Version string + Checksum string // MD5 of the retained .exmodz bytes (md5File) +} + +// marshalMergedFingerprint renders f deterministically: encoding/json +// marshals struct fields in declaration order (not sorted) and preserves +// slice order exactly, so the same MergedFingerprint value always produces +// byte-identical output - the property mergedFingerprintsEqual depends on. +// +// A nil Mods is normalized to an empty (non-nil) slice first: encoding/json +// marshals a nil slice as `null` but an empty slice as `[]` - two DIFFERENT +// byte sequences for what must count as the same "zero contributing mods" +// state (e.g. a freshly-built "current" fingerprint via `var mods []T` +// compared against a previously-stored marker written some other way). +// Caught by extraction-verification (a scratch test comparing the two +// literally failed before this normalization was added) - without it, a +// profile with zero enabled exmodz mods could spuriously flip between +// "stale"/"not stale" depending on which code path happened to build each +// side's slice. +func marshalMergedFingerprint(f MergedFingerprint) ([]byte, error) { + if f.Mods == nil { + f.Mods = []MergedFingerprintEntry{} + } + return json.Marshal(f) +} + +// mergedFingerprintsEqual reports whether a and b describe the same merge +// inputs, by comparing their marshaled bytes - exactly what "compare +// against the stored marker" needs, since the marker itself IS the +// marshaled form. +func mergedFingerprintsEqual(a, b MergedFingerprint) (bool, error) { + aBytes, err := marshalMergedFingerprint(a) + if err != nil { + return false, err + } + bBytes, err := marshalMergedFingerprint(b) + if err != nil { + return false, err + } + return bytes.Equal(aBytes, bBytes), nil +} + +// enabledExmodzSources returns every enabled mod's retained .exmodz files +// for game+profileName, in PROFILE LOAD ORDER (the merge-application order, +// #197 design) - the exact input MergeCompile needs. Only files that were +// actually retained (cache.RetainedSourceName present in the mod's cache +// entry) count: a mod's plain .pak files, or a mod whose ingest never got +// far enough to retain anything, contribute nothing. A mod's OWN FileIDs +// are walked (not the whole cache directory) because a download-compiled +// entry's retained-source name is keyed by a real DownloadableFile.ID, +// while an import-compiled entry's is keyed by its own archive filename +// (see Task 2/3's ingest branches) - FileIDs is the one list that already +// carries whichever identity applies, for either origin. +func (s *Service) enabledExmodzSources(game *domain.Game, profileName string) ([]source.MergeSource, error) { + mods, err := s.GetInstalledModsInProfileOrder(game.ID, profileName) + if err != nil { + return nil, fmt.Errorf("loading profile mods: %w", err) + } + + gameCache := s.GetGameCache(game) + var sources []source.MergeSource + for _, mod := range mods { + if !mod.Enabled { + continue + } + for _, fileID := range mod.FileIDs { + retainedPath := gameCache.GetFilePath(game.ID, mod.SourceID, mod.ID, mod.Version, cache.RetainedSourceName(fileID)) + if _, statErr := os.Stat(retainedPath); statErr != nil { + continue // not a retained exmodz file (a plain .pak's fileID, or nothing ingested) + } + sources = append(sources, source.MergeSource{ + ModRef: mod.SourceID + ":" + mod.ID, + ExmodzPath: retainedPath, + }) + } + } + return sources, nil +} +``` + +Add `"os"` to the import block (used by `os.Stat`). + +- [ ] **Step 6: Run the tests, confirm they pass** + +Run: `go test ./internal/storage/cache/... ./internal/core/... -run 'TestCache_MergeFingerprintPath|TestMergedFingerprint' -v` +Expected: all PASS. + +- [ ] **Step 7: Write and run the `enabledExmodzSources` black-box test** + +Replace the placeholder `internal/core/merged_pak_test.go` (from Step 1) with: + +```go +package core_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/core" + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/DonovanMods/linux-mod-manager/internal/storage/cache" + "github.com/stretchr/testify/require" +) + +// TestEnabledExmodzSources_OrderMatchesProfileLoadOrderAndSkipsDisabled +// proves enabledExmodzSources returns retained exmodz files in PROFILE +// LOAD ORDER (merge-application order), skips disabled mods entirely, and +// skips a mod's fileIDs that have no retained source (a plain .pak). +func TestEnabledExmodzSources_OrderMatchesProfileLoadOrderAndSkipsDisabled(t *testing.T) { + svc := newFlowsTestService(t) + game := &domain.Game{ID: "icarus", ModPath: t.TempDir(), DeployMode: domain.DeployCompile} + require.NoError(t, svc.AddGame(game)) + + gameCache := svc.GetGameCache(game) + + seedMod := func(sourceID, modID, version string, fileIDs []string, enabled bool) { + for _, fileID := range fileIDs { + require.NoError(t, gameCache.Store(game.ID, sourceID, modID, version, cache.RetainedSourceName(fileID), []byte("exmodz-"+modID+"-"+fileID))) + } + require.NoError(t, svc.SaveInstalledMod(&domain.InstalledMod{ + Mod: domain.Mod{ID: modID, SourceID: sourceID, Name: modID, Version: version, GameID: game.ID}, + ProfileName: "default", + Enabled: enabled, + FileIDs: fileIDs, + UpdatePolicy: domain.UpdateNotify, + })) + } + + // mixedMod has one exmodz fileID and one plain-pak fileID (no retained + // source for the latter) - only the exmodz one should be included. + seedMod("icarus", "second-mod", "1.0", []string{"exmodz-file", "pak-file"}, true) + seedMod("icarus", "first-mod", "1.0", []string{"exmodz-file"}, true) + seedMod("icarus", "disabled-mod", "1.0", []string{"exmodz-file"}, false) + + pm := svc.NewProfileManager() + _, err := pm.Create(game.ID, "default") + require.NoError(t, err) + // Profile load order: first-mod, then second-mod (disabled-mod + // intentionally omitted - membership in Profile.Mods, not just an + // Enabled DB row, is what GetInstalledModsInProfileOrder requires). + require.NoError(t, pm.UpsertMod(game.ID, "default", domain.ModReference{SourceID: "icarus", ModID: "first-mod", Version: "1.0", FileIDs: []string{"exmodz-file"}})) + require.NoError(t, pm.UpsertMod(game.ID, "default", domain.ModReference{SourceID: "icarus", ModID: "second-mod", Version: "1.0", FileIDs: []string{"exmodz-file", "pak-file"}})) + require.NoError(t, pm.UpsertMod(game.ID, "default", domain.ModReference{SourceID: "icarus", ModID: "disabled-mod", Version: "1.0", FileIDs: []string{"exmodz-file"}})) + + sources, err := svc.EnabledExmodzSourcesForTest(game, "default") + require.NoError(t, err) + require.Len(t, sources, 2, "disabled-mod excluded; second-mod's plain-pak fileID excluded") + require.Equal(t, "icarus:first-mod", sources[0].ModRef) + require.Equal(t, "icarus:second-mod", sources[1].ModRef) + + data, err := os.ReadFile(sources[0].ExmodzPath) + require.NoError(t, err) + require.Equal(t, "exmodz-first-mod-exmodz-file", string(data)) +} + +var _ = context.Background // keep context import if a future case needs ctx +var _ = filepath.Join // keep filepath import if a future case needs it +``` + +**`enabledExmodzSources` is unexported** — this black-box test needs an exported test seam. Add ONE tiny exported wrapper to `internal/core/merged_pak.go`, directly below `enabledExmodzSources`: + +```go +// EnabledExmodzSourcesForTest exposes enabledExmodzSources to external +// (core_test package) tests - the method itself stays unexported since it +// is an internal implementation detail of syncMergedPak (Task 6), not part +// of Service's public API. +func (s *Service) EnabledExmodzSourcesForTest(game *domain.Game, profileName string) ([]source.MergeSource, error) { + return s.enabledExmodzSources(game, profileName) +} +``` + +(This mirrors an existing pattern already used elsewhere in this codebase for white-box-only helpers that still need black-box test coverage — if a `*ForTest`-style export doesn't already appear anywhere via `grep -rn "ForTest" internal/core/*.go`, prefer instead moving JUST this one test into a new `internal/core/merged_pak_internal_test.go` `package core` file, calling `s.enabledExmodzSources` directly with no exported wrapper at all — simpler, and avoids adding test-only surface to `Service`. Either approach is correct; the internal-test-file route is preferred if there's no existing `*ForTest` precedent to stay consistent with.) + +Remove the unused `var _ = context.Background` / `var _ = filepath.Join` lines above if the final test file doesn't need those imports at all (it doesn't, in the version shown — they were placeholders for exactly this "which route did you take" branch; delete `"context"` and `"path/filepath"` from the import block too if going the internal-test-file route, since neither `context` nor `filepath` is used in that case). + +Run: `go test ./internal/core/... -run 'TestEnabledExmodzSources' -v` +Expected: PASS. + +- [ ] **Step 8: Run the full build and suite** + +Run: `go build ./... 2>&1 | tail -40` +Expected: same confined failures as Task 1 Step 8 noted (`internal/core/importer.go`... wait, Task 2/3 already fixed those — expected failures now are ONLY in `internal/core/updater.go`, fixed in Task 6/9). + +Run: `go test ./internal/storage/cache/... ./internal/core/... -v 2>&1 | tail -80` (skip the rest of the repo until `updater.go` is fixed in Task 6). + +- [ ] **Step 9: Commit** + +```bash +git add internal/domain/mod.go internal/storage/cache/cache.go internal/storage/cache/cache_test.go internal/core/merged_pak.go internal/core/merged_pak_test.go internal/core/merged_pak_internal_test.go +git commit -m "feat: MergedFingerprint type + enabledExmodzSources (#197)" +``` + +### Task 6: `Service.syncMergedPak` — regenerate-if-stale engine + +**Files:** + +- Modify: `internal/core/merged_pak.go` +- Test: `internal/core/merged_pak_test.go` + +**Interfaces:** + +- Consumes: `enabledExmodzSources` (Task 5); `MergedFingerprint`/`marshalMergedFingerprint`/`mergedFingerprintsEqual` (Task 5); `mergeCompilerSourceForGame` (Task 2); `resolveBasePak`/`basePakIndexHash` (#196, kept by Task 4); `md5File` (#196); `cache.MergeFingerprintPath` (Task 5); `prepareUnseededStaging`/`commitStagedCache` (#196, unchanged); `s.GetInstallerForProfile` (existing); `Installer.Install`/`Uninstall` (existing, unchanged). +- Produces: `Service.syncMergedPak(ctx context.Context, game *domain.Game, profileName string) (warnings []string, err error)` — consumed by Task 7/8's hook call sites and Task 9's `ApplyMergedPakRegen`. + +- [ ] **Step 1: Write the failing tests** + +Append to `internal/core/merged_pak_test.go`: + +```go +// newMergedPakTestGame builds a DeployCompile game with a registered merge +// compiler and an installed base pak - shared setup for syncMergedPak +// tests. Returns the service, game, and the base pak's own path (so a test +// can rewrite it to simulate a base-pak refresh). +func newMergedPakTestGame(t *testing.T) (*core.Service, *domain.Game, string) { + t.Helper() + + installDir := t.TempDir() + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + writeFakeBasePak(t, basePak) + + svc := newFlowsTestService(t) + src := &fakeCompilerSource{} + svc.RegisterSource(src) + + game := &domain.Game{ + ID: "icarus", InstallPath: installDir, ModPath: t.TempDir(), + DeployMode: domain.DeployCompile, LinkMethod: domain.LinkCopy, + SourceIDs: map[string]string{"fake-compiler": "external-icarus-id"}, + } + require.NoError(t, svc.AddGame(game)) + + pm := svc.NewProfileManager() + _, err := pm.Create(game.ID, "default") + require.NoError(t, err) + + return svc, game, basePak +} + +// seedEnabledExmodzMod installs an ENABLED mod with a retained exmodz file, +// via svc.SaveInstalledMod + profile UpsertMod (matching the real ingest +// shape Task 2/3 produce - a cache entry with a retained source and no +// deployment members). +func seedEnabledExmodzMod(t *testing.T, svc *core.Service, game *domain.Game, sourceID, modID, version, fileID string, exmodzContent []byte) { + t.Helper() + gameCache := svc.GetGameCache(game) + require.NoError(t, gameCache.Store(game.ID, sourceID, modID, version, cache.RetainedSourceName(fileID), exmodzContent)) + require.NoError(t, svc.SaveInstalledMod(&domain.InstalledMod{ + Mod: domain.Mod{ID: modID, SourceID: sourceID, Name: modID, Version: version, GameID: game.ID}, + ProfileName: "default", + Enabled: true, + FileIDs: []string{fileID}, + UpdatePolicy: domain.UpdateNotify, + })) + pm := svc.NewProfileManager() + require.NoError(t, pm.UpsertMod(game.ID, "default", domain.ModReference{SourceID: sourceID, ModID: modID, Version: version, FileIDs: []string{fileID}})) +} + +// TestSyncMergedPak_GeneratesAndDeploys is the happy path: one enabled +// exmodz mod, no merged pak yet - syncMergedPak must generate one and +// deploy it into the game directory. +func TestSyncMergedPak_GeneratesAndDeploys(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-exmodz-bytes")) + + warnings, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + require.Empty(t, warnings) + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + data, err := os.ReadFile(deployedPath) + require.NoError(t, err) + require.Equal(t, "bear-exmodz-bytes", string(data), "fakeCompilerSource's MergeCompile concatenates source bytes - see its own definition") +} + +// TestSyncMergedPak_NoOpWhenUnchanged proves the fingerprint gate actually +// gates: calling syncMergedPak twice with nothing changed must not +// recompile (fakeCompilerSource.compileCalls stays at 1). +func TestSyncMergedPak_NoOpWhenUnchanged(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-exmodz-bytes")) + + _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + + src, ok := svc.GetSourceForTest("fake-compiler").(*fakeCompilerSource) + require.True(t, ok) + require.Equal(t, 1, src.compileCalls) + + _, err = svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + require.Equal(t, 1, src.compileCalls, "an unchanged fingerprint must not trigger a second merge") +} + +// TestSyncMergedPak_RegeneratesOnModEnable proves enabling a SECOND mod +// (the mod-set changing) triggers regeneration. +func TestSyncMergedPak_RegeneratesOnModEnable(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + + _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "wolf-mount", "1.0", "exmodz-file", []byte("wolf-bytes")) + + warnings, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + require.Empty(t, warnings) + + src, ok := svc.GetSourceForTest("fake-compiler").(*fakeCompilerSource) + require.True(t, ok) + require.Equal(t, 2, src.compileCalls, "a mod-set change must trigger a second merge") + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + data, err := os.ReadFile(deployedPath) + require.NoError(t, err) + require.Equal(t, "bear-byteswolf-bytes", string(data), "the merged pak must now reflect BOTH mods") +} + +// TestSyncMergedPak_ZeroEnabledMods_UninstallsExistingPak proves the +// uninstall-to-zero case: disabling the LAST enabled exmodz mod must +// remove any previously-deployed merged pak from the game directory. +func TestSyncMergedPak_ZeroEnabledMods_UninstallsExistingPak(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + + _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + _, err = os.Stat(deployedPath) + require.NoError(t, err, "precondition: the merged pak must exist before disabling") + + require.NoError(t, svc.SetModEnabled("fake-compiler", "bear-mount", game.ID, "default", false)) + + _, err = svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + + _, err = os.Stat(deployedPath) + require.True(t, os.IsNotExist(err), "disabling the last exmodz mod must remove the deployed merged pak") +} + +// TestSyncMergedPak_RegeneratesOnBaseHashChange proves a base-pak refresh +// (the "Friday problem", generalized from #196 to the merged model) still +// triggers regeneration. +func TestSyncMergedPak_RegeneratesOnBaseHashChange(t *testing.T) { + svc, game, basePak := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + + _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + + // Rewrite the base pak with different content - a new IndexHash. + writeFakeBasePakWithTable(t, basePak, map[string][]byte{"AI/D_Other.json": []byte(`{"Rows":[{"Name":"x","V":1}]}`)}) + + _, err = svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + + src, ok := svc.GetSourceForTest("fake-compiler").(*fakeCompilerSource) + require.True(t, ok) + require.Equal(t, 2, src.compileCalls, "a base pak change must trigger a second merge") +} + +// TestSyncMergedPak_NonCompileGame_NoOp: a DeployExtract/DeployCopy game has +// no merged-pak concept at all - syncMergedPak must no-op unconditionally +// (cheap enough to call from every mutation flow regardless of game type). +func TestSyncMergedPak_NonCompileGame_NoOp(t *testing.T) { + svc := newFlowsTestService(t) + game := &domain.Game{ID: "skyrim-se", ModPath: t.TempDir(), DeployMode: domain.DeployExtract} + require.NoError(t, svc.AddGame(game)) + + warnings, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + require.Empty(t, warnings) +} + +// TestSyncMergedPak_AssetCollisionWarningSurfaces proves MergeCompile's own +// warnings (Task 1) propagate all the way out of syncMergedPak. +func TestSyncMergedPak_AssetCollisionWarningSurfaces(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + src, ok := svc.GetSourceForTest("fake-compiler").(*fakeCompilerSource) + require.True(t, ok) + src.mergeWarnings = []string{"asset collision: fixture warning"} + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + + warnings, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + require.Equal(t, []string{"asset collision: fixture warning"}, warnings) +} +``` + +This introduces THREE new small test seams the fake/harness must gain: + +1. `fakeCompilerSource.mergeWarnings []string` — its `MergeCompile` returns this slice instead of always `nil`. Update the fake (already changed in Task 2 Step 1) in `internal/core/service_icarus_compile_test.go`: + +```go +func (s *fakeCompilerSource) MergeCompile(ctx context.Context, basePakPath string, sources []source.MergeSource, outputPath string) ([]string, error) { + s.compileCalls++ + var out []byte + for _, src := range sources { + data, err := os.ReadFile(src.ExmodzPath) + if err != nil { + return nil, err + } + out = append(out, data...) + } + return s.mergeWarnings, os.WriteFile(outputPath, out, 0o644) +} +``` + +Add `mergeWarnings []string` to the `fakeCompilerSource` struct. + +2. `Service.SyncMergedPakForTest` — thin exported wrapper over `syncMergedPak`, same rationale/pattern as Task 5 Step 7's `EnabledExmodzSourcesForTest` (or, per that step's note, skip the wrapper and put these tests in the internal white-box test file instead — pick whichever route Task 5 used, for consistency, and apply it here too). + +3. `Service.GetSourceForTest` — a tiny exported wrapper over `s.registry.Get`, needed because these tests must reach into the `fakeCompilerSource` to assert `compileCalls`/set `mergeWarnings`, and `Service` has no existing public accessor for a registered source by ID (`GetSource` DOES already exist — `internal/core/service.go:108`, `func (s *Service) GetSource(id string) (source.ModSource, error)` — **use that directly, no new wrapper needed**; the test snippets above should read `svc.GetSource("fake-compiler")` (handling the returned error) rather than a fictitious `GetSourceForTest` — this was written assuming no existing accessor; there IS one, use it and drop this wrapper entirely). + +Fix the test snippets above: replace every `svc.GetSourceForTest("fake-compiler").(*fakeCompilerSource)` with: + +```go + srcRaw, err := svc.GetSource("fake-compiler") + require.NoError(t, err) + src, ok := srcRaw.(*fakeCompilerSource) + require.True(t, ok) +``` + +(inline this 4-line block wherever `svc.GetSourceForTest(...)` appears above). + +Also add `writeFakeBasePakWithTable` (a variant of the existing `writeFakeBasePak` helper that lets a test control table content, needed for `TestSyncMergedPak_RegeneratesOnBaseHashChange`) to `internal/core/service_icarus_compile_test.go` next to `writeFakeBasePak`: + +```go +func writeFakeBasePakWithTable(t *testing.T, path string, tables map[string][]byte) { + t.Helper() + w, err := unrealpak.Create(path) + require.NoError(t, err) + for mountPath, data := range tables { + require.NoError(t, w.AddFile(mountPath, data)) + } + require.NoError(t, w.Close()) +} +``` + +Add `"github.com/DonovanMods/linux-mod-manager/internal/unrealpak"` to that file's imports if not already present. + +- [ ] **Step 2: Run the tests, confirm they fail to compile** + +Run: `go test ./internal/core/... -run 'TestSyncMergedPak' -v` +Expected: build failure — `undefined: (*core.Service).SyncMergedPakForTest` (or the internal-test-file equivalent), `mergeWarnings` undefined on `fakeCompilerSource`. + +- [ ] **Step 3: Implement `syncMergedPak`** + +Append to `internal/core/merged_pak.go`: + +```go +// syncMergedPak regenerates game+profileName's merged pak if its recorded +// fingerprint no longer matches the CURRENT enabled-mod set/order/versions/ +// base pak (#197). Cheap when nothing changed: the fast path is one +// directory read (enabledExmodzSources), one base-pak footer read +// (basePakIndexHash - never the pak's full content), and N small MD5s +// (md5File over each retained .exmodz - real files here are small, see +// #175's own research on real base-table sizes), then a byte comparison. +// Safe to call unconditionally from ANY mutation flow regardless of game +// type - it no-ops immediately for a non-DeployCompile game. +// +// Zero enabled exmodz sources uninstalls any existing merged pak instead of +// generating an empty one (#197 design decision 2's "uninstall-to-zero" +// requirement) - Installer.Uninstall on the synthetic merged-pak mod is +// idempotent when there is nothing deployed (linker.Undeploy tolerates an +// already-absent path, matching every other uninstall in this codebase), +// so calling it unconditionally here is safe even when no pak was ever +// generated. +func (s *Service) syncMergedPak(ctx context.Context, game *domain.Game, profileName string) (warnings []string, err error) { + if game.DeployMode != domain.DeployCompile { + return nil, nil + } + + sources, err := s.enabledExmodzSources(game, profileName) + if err != nil { + return nil, fmt.Errorf("listing enabled exmodz mods: %w", err) + } + + gameCache := s.GetGameCache(game) + syntheticMod := &domain.Mod{ID: mergedPakModID, SourceID: domain.SourceMerged, Version: mergedPakVersion, GameID: game.ID} + + installer, err := s.GetInstallerForProfile(game, profileName) + if err != nil { + return nil, err + } + + if len(sources) == 0 { + if uerr := installer.Uninstall(ctx, game, syntheticMod, profileName); uerr != nil { + return nil, fmt.Errorf("removing merged pak: %w", uerr) + } + if derr := gameCache.Delete(game.ID, domain.SourceMerged, mergedPakModID, mergedPakVersion); derr != nil { + return nil, fmt.Errorf("clearing merged pak cache entry: %w", derr) + } + return nil, nil + } + + basePakPath, err := resolveBasePak(game) + if err != nil { + return nil, err + } + liveHash, err := basePakIndexHash(basePakPath) + if err != nil { + return nil, fmt.Errorf("reading base pak for merge fingerprint: %w", err) + } + + current := MergedFingerprint{BaseIndexHash: liveHash} + for _, src := range sources { + sum, herr := md5File(src.ExmodzPath) + if herr != nil { + return nil, fmt.Errorf("hashing %s: %w", src.ExmodzPath, herr) + } + sourceID, modID, _ := strings.Cut(src.ModRef, ":") + current.Mods = append(current.Mods, MergedFingerprintEntry{ + SourceID: sourceID, ModID: modID, Checksum: sum, + }) + } + // Version is not carried on source.MergeSource (it only needs ModRef + + // ExmodzPath for the merge itself) - resolved separately here so + // enabledExmodzSources' own signature stays minimal. Re-fetching the + // installed mods once more is cheap (small profiles) and keeps + // enabledExmodzSources' contract focused on ONE job. + mods, err := s.GetInstalledModsInProfileOrder(game.ID, profileName) + if err != nil { + return nil, fmt.Errorf("loading profile mods: %w", err) + } + versionByRef := make(map[string]string, len(mods)) + for _, m := range mods { + versionByRef[m.SourceID+":"+m.ID] = m.Version + } + for i, src := range sources { + current.Mods[i].Version = versionByRef[src.ModRef] + } + + cachePath := gameCache.ModPath(game.ID, domain.SourceMerged, mergedPakModID, mergedPakVersion) + if stored, ok := readMergedFingerprint(cachePath); ok { + if eq, eqErr := mergedFingerprintsEqual(current, stored); eqErr == nil && eq { + return nil, nil // fast path: nothing changed + } + } + + mc, err := s.mergeCompilerSourceForGame(game.ID) + if err != nil { + return nil, err + } + + stagePath := cachePath + ".staging" + if err := os.RemoveAll(stagePath); err != nil { + return nil, fmt.Errorf("clearing merged pak staging: %w", err) + } + if err := os.MkdirAll(stagePath, 0755); err != nil { + return nil, fmt.Errorf("preparing merged pak staging: %w", err) + } + defer os.RemoveAll(stagePath) //nolint:errcheck + + outputPath := filepath.Join(stagePath, mergedPakFileName) + mergeWarnings, err := mc.MergeCompile(ctx, basePakPath, sources, outputPath) + if err != nil { + return nil, fmt.Errorf("merging %d exmodz mod(s): %w", len(sources), err) + } + warnings = mergeWarnings + + fingerprintBytes, err := marshalMergedFingerprint(current) + if err != nil { + return warnings, fmt.Errorf("encoding merge fingerprint: %w", err) + } + if err := os.WriteFile(cache.MergeFingerprintPath(stagePath), fingerprintBytes, 0644); err != nil { + return warnings, fmt.Errorf("writing merge fingerprint: %w", err) + } + + if err := commitStagedCache(cachePath, stagePath); err != nil { + return warnings, err + } + + if err := installer.Install(ctx, game, syntheticMod, profileName); err != nil { + return warnings, fmt.Errorf("deploying merged pak: %w", err) + } + return warnings, nil +} + +// readMergedFingerprint reads and decodes cachePath's stored merge +// fingerprint marker, if any. ok is false when no cache entry/marker +// exists yet (first-ever merge for this profile) or the marker is +// unreadable/corrupt - both degrade to "regenerate", never a crash or a +// false "unchanged". +func readMergedFingerprint(cachePath string) (fp MergedFingerprint, ok bool) { + data, err := os.ReadFile(cache.MergeFingerprintPath(cachePath)) + if err != nil { + return MergedFingerprint{}, false + } + if err := json.Unmarshal(data, &fp); err != nil { + return MergedFingerprint{}, false + } + return fp, true +} +``` + +Add `"context"`, `"os"`, `"path/filepath"`, `"strings"` to `merged_pak.go`'s import block (`json` and `bytes` are already there from Task 5). + +- [ ] **Step 4: Add the `SyncMergedPakForTest` wrapper (or internal test file — match Task 5's choice)** + +If Task 5 used the exported-wrapper route: + +```go +// SyncMergedPakForTest exposes syncMergedPak to external (core_test +// package) tests - see enabledExmodzSources/EnabledExmodzSourcesForTest's +// identical rationale. +func (s *Service) SyncMergedPakForTest(ctx context.Context, game *domain.Game, profileName string) ([]string, error) { + return s.syncMergedPak(ctx, game, profileName) +} +``` + +If Task 5 used the internal-white-box-test-file route instead, move this task's new tests into that same `internal/core/merged_pak_internal_test.go` file and call `s.syncMergedPak(...)` directly — no wrapper. + +- [ ] **Step 5: Run the tests, confirm they pass** + +Run: `go test ./internal/core/... -run 'TestSyncMergedPak' -v` +Expected: all 7 PASS. + +- [ ] **Step 6: Run the full build** + +Run: `go build ./... 2>&1 | tail -40` +Expected: `internal/core/updater.go` is now the ONLY remaining broken file (`CheckBaseStaleness`/`ApplyRecompile` still reference removed `BaseIndexHashes`) — fixed in Task 9. + +- [ ] **Step 7: Commit** + +```bash +git add internal/core/merged_pak.go internal/core/merged_pak_test.go internal/core/service_icarus_compile_test.go +git commit -m "feat: Service.syncMergedPak - regenerate-if-stale engine (#197)" +``` + +### Task 7: Wire `syncMergedPak` into `EnableMod`/`DisableMod`/`UninstallMod`/`DeployProfile` + +**Files:** + +- Modify: `internal/core/flows.go` (4 call sites, listed below) +- Test: `internal/core/flows_test.go` (or a new `internal/core/merged_pak_hooks_test.go` — either is fine; this plan uses a new file to keep the diff to `flows.go` reviewable independently of a large pre-existing test file) + +**Interfaces:** + +- Consumes: `Service.syncMergedPak` (Task 6). +- Produces: nothing new — this task is pure wiring. + +Every insertion below follows the IDENTICAL shape: call `syncMergedPak`, fold its warnings into the function's own existing diagnostics field, fold a hard error into the SAME non-fatal-Notes convention each function already uses for its OTHER best-effort side effects (never let a merged-pak sync failure turn an otherwise-successful enable/disable/uninstall/deploy into a hard error — the mod-level operation already succeeded; the merged pak catching up is a courtesy, and `lmm update`/`lmm verify` are the safety net if it doesn't). + +- [ ] **Step 1: Write the failing tests** + +Create `internal/core/merged_pak_hooks_test.go`: + +```go +package core_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/stretchr/testify/require" +) + +// TestEnableMod_SyncsMergedPak proves enabling an exmodz mod deploys the +// merged pak without a separate `lmm update` step. +func TestEnableMod_SyncsMergedPak(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + require.NoError(t, svc.SetModEnabled("fake-compiler", "bear-mount", game.ID, "default", false)) + + _, err := svc.EnableMod(context.Background(), game, "default", "fake-compiler", "bear-mount") + require.NoError(t, err) + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + _, err = os.Stat(deployedPath) + require.NoError(t, err, "EnableMod must sync the merged pak, not just this mod's own (empty) cache entry") +} + +// TestDisableMod_SyncsMergedPak_RemovesWhenLastModDisabled proves disabling +// the LAST enabled exmodz mod removes the merged pak. +func TestDisableMod_SyncsMergedPak_RemovesWhenLastModDisabled(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + _, err = os.Stat(deployedPath) + require.NoError(t, err) + + _, err = svc.DisableMod(context.Background(), game, "default", "fake-compiler", "bear-mount") + require.NoError(t, err) + + _, err = os.Stat(deployedPath) + require.True(t, os.IsNotExist(err), "DisableMod must sync the merged pak, removing it once the last exmodz mod is disabled") +} + +// TestUninstallMod_SyncsMergedPak_RemovesWhenLastModUninstalled mirrors +// the disable case for a full uninstall. +func TestUninstallMod_SyncsMergedPak_RemovesWhenLastModUninstalled(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + + _, err = svc.UninstallMod(context.Background(), game, "default", "fake-compiler", "bear-mount", core.UninstallOptions{}) + require.NoError(t, err) + + _, err = os.Stat(deployedPath) + require.True(t, os.IsNotExist(err), "UninstallMod must sync the merged pak") +} + +// TestDeployProfile_SyncsMergedPak proves a full `lmm deploy` also +// generates the merged pak (the pre-existing per-mod loop deploys zero +// files for an exmodz mod's own cache entry - Tasks 2/3 - so without this +// hook a fresh deploy would silently produce no merged pak at all). +func TestDeployProfile_SyncsMergedPak(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + + _, err := svc.DeployProfile(context.Background(), game, "default", core.DeployOptions{}, nil) + require.NoError(t, err) + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + data, err := os.ReadFile(deployedPath) + require.NoError(t, err) + require.Equal(t, "bear-bytes", string(data)) +} +``` + +Add `"github.com/DonovanMods/linux-mod-manager/internal/core"` to the imports (needed for `core.UninstallOptions{}`/`core.DeployOptions{}`). + +- [ ] **Step 2: Run the tests, confirm they fail** + +Run: `go test ./internal/core/... -run 'TestEnableMod_SyncsMergedPak|TestDisableMod_SyncsMergedPak|TestUninstallMod_SyncsMergedPak|TestDeployProfile_SyncsMergedPak' -v` +Expected: FAIL (not build failure this time — `EnableMod`/etc. already compile and run, they just don't call `syncMergedPak` yet, so no merged pak is ever deployed). + +- [ ] **Step 3: Wire `EnableMod`** + +`internal/core/flows.go`, `EnableMod` (ends around line 93-94 with `result.Changed = true` then `return result, nil`). Insert immediately before the final `return result, nil`: + +```go + if syncWarnings, syncErr := s.syncMergedPak(ctx, game, profileName); syncErr != nil { + result.Notes = append(result.Notes, fmt.Sprintf("Warning: could not sync merged pak: %v", syncErr)) + } else { + for _, w := range syncWarnings { + result.Notes = append(result.Notes, "Warning: "+w) + } + } + + result.Changed = true + return result, nil +``` + +(replacing the existing bare `result.Changed = true` / `return result, nil` pair with this — the sync call runs BEFORE `result.Changed = true` is set here only because that's where the existing lines already sat; functionally the ordering relative to `Changed` doesn't matter.) + +- [ ] **Step 4: Wire `DisableMod`** + +`internal/core/flows.go`, `DisableMod`'s MAIN path (not the already-disabled self-heal early return above it) ends with `result.Changed = true` / `return result, nil` (around line 165-166). Apply the identical insertion: + +```go + if syncWarnings, syncErr := s.syncMergedPak(ctx, game, profileName); syncErr != nil { + result.Notes = append(result.Notes, fmt.Sprintf("Warning: could not sync merged pak: %v", syncErr)) + } else { + for _, w := range syncWarnings { + result.Notes = append(result.Notes, "Warning: "+w) + } + } + + result.Changed = true + return result, nil +``` + +Do NOT add this to the already-disabled self-heal branch (`if !mod.Enabled { ... }`, earlier in the function) — nothing about the enabled-mod-set changed there, so there is nothing to sync. + +- [ ] **Step 5: Wire `UninstallMod`** + +`internal/core/flows.go`, `UninstallMod` ends with `return result, nil` (around line 294, per this task's own investigation). Read the ~15 lines immediately before that return first (`grep -n "^func (s \*Service) UninstallMod" -A 260 internal/core/flows.go | tail -40`) to confirm what `result` variable is in scope and its exact type (`*UninstallResult`, `Notes []string`) before inserting — then insert the same 7-line block immediately before that final `return result, nil`: + +```go + if syncWarnings, syncErr := s.syncMergedPak(ctx, game, profileName); syncErr != nil { + result.Notes = append(result.Notes, fmt.Sprintf("Warning: could not sync merged pak: %v", syncErr)) + } else { + for _, w := range syncWarnings { + result.Notes = append(result.Notes, "Warning: "+w) + } + } + + return result, nil +``` + +- [ ] **Step 6: Wire `DeployProfile`** + +`internal/core/flows.go:1810-1816` — the existing profile-overrides step is the natural "runs once per DeployProfile call, after all mod files are on disk" anchor (this task's own investigation identified it as the ONLY existing whole-profile step in this function). Insert immediately AFTER that block, still BEFORE the `for _, w := range deferredWarnings { emit(w) }` loop: + +```go + if syncWarnings, syncErr := s.syncMergedPak(ctx, game, profileName); syncErr != nil { + msg := fmt.Sprintf("syncing merged pak: %v", syncErr) + result.Warnings = append(result.Warnings, msg) + emit(DeployProgress{Phase: DeployWarning, Detail: msg}) + } else { + for _, w := range syncWarnings { + result.Warnings = append(result.Warnings, w) + emit(DeployProgress{Phase: DeployWarning, Detail: w}) + } + } +``` + +(`DeployResult.Warnings` and `DeployWarning`/`DeployProgress`/`emit` are all pre-existing in this function's scope — no new types needed.) + +- [ ] **Step 7: Run the tests, confirm they pass** + +Run: `go test ./internal/core/... -run 'TestEnableMod_SyncsMergedPak|TestDisableMod_SyncsMergedPak|TestUninstallMod_SyncsMergedPak|TestDeployProfile_SyncsMergedPak' -v` +Expected: all 4 PASS. + +- [ ] **Step 8: Run the full core suite** + +Run: `go test ./internal/core/... 2>&1 | tail -100` +Expected: all PASS except `updater.go`'s own tests (Task 9). Pay particular attention to any EXISTING `TestEnableMod_*`/`TestDisableMod_*`/`TestDeployProfile_*` test for a NON-DeployCompile game — `syncMergedPak`'s own `game.DeployMode != domain.DeployCompile` no-op guard (Task 6) must make this wiring invisible to every one of them; if any fails, the guard isn't firing correctly. + +- [ ] **Step 9: Commit** + +```bash +git add internal/core/flows.go internal/core/merged_pak_hooks_test.go +git commit -m "feat: sync merged pak on enable/disable/uninstall/deploy (#197)" +``` + +### Task 8: Wire `syncMergedPak` into `ApplyProfileSwitch`/`ApplyUpdate`/`ApplyInstall` + new `Service.ReorderProfileMods` + +**Files:** + +- Modify: `internal/core/flows.go` (`ApplyProfileSwitch`, `ApplyUpdate`, `ApplyInstall`) +- Modify: `internal/core/profile.go` (new `Service.ReorderProfileMods` — check this file for where `Service`-level profile wrappers already live, e.g. near `NewProfileManager`; if `Service` has no existing profile-wrapper methods in `profile.go`, add it to `internal/core/flows.go` instead, next to `EnableMod`/`DisableMod`) +- Modify: `cmd/lmm/profile.go:831` (call `Service.ReorderProfileMods` instead of `pm.ReorderMods` directly) +- Modify: `internal/tui/service_core.go:978` (same) +- Test: `internal/core/merged_pak_hooks_test.go` (append) + +**Interfaces:** + +- Consumes: `Service.syncMergedPak` (Task 6); `ProfileManager.ReorderMods(gameID, profileName string, mods []domain.ModReference) error` (existing, unchanged). +- Produces: `Service.ReorderProfileMods(gameID, profileName string, mods []domain.ModReference) error` — consumed by `cmd/lmm/profile.go` and `internal/tui/service_core.go`. + +- [ ] **Step 1: Write the failing tests** + +Append to `internal/core/merged_pak_hooks_test.go`: + +```go +// TestApplyProfileSwitch_SyncsMergedPakForToProfile proves switching TO a +// profile with enabled exmodz mods deploys ITS merged pak (plan.To, not +// plan.From). +func TestApplyProfileSwitch_SyncsMergedPakForToProfile(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + pm := svc.NewProfileManager() + _, err := pm.Create(game.ID, "other") + require.NoError(t, err) + + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + // Move the mod's profile membership to "other" too, so switching there + // has something enabled to merge. + require.NoError(t, pm.UpsertMod(game.ID, "other", domain.ModReference{SourceID: "fake-compiler", ModID: "bear-mount", Version: "1.0", FileIDs: []string{"exmodz-file"}})) + mod, err := svc.GetInstalledMod("fake-compiler", "bear-mount", game.ID, "default") + require.NoError(t, err) + mod.ProfileName = "other" + require.NoError(t, svc.SaveInstalledMod(mod)) + + plan := &core.SwitchPlan{From: "default", To: "other"} + _, err = svc.ApplyProfileSwitch(context.Background(), game, plan, nil) + require.NoError(t, err) + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + _, err = os.Stat(deployedPath) + require.NoError(t, err, "ApplyProfileSwitch must sync the merged pak for the TO profile") +} + +// TestReorderProfileMods_SyncsMergedPak proves a load-order change (a +// documented regeneration trigger) actually reaches the merged pak. +func TestReorderProfileMods_SyncsMergedPak(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-a", []byte("A")) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "wolf-mount", "1.0", "exmodz-b", []byte("B")) + _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + before, err := os.ReadFile(deployedPath) + require.NoError(t, err) + require.Equal(t, "AB", string(before)) + + // Swap load order: wolf-mount now first. + err = svc.ReorderProfileMods(game.ID, "default", []domain.ModReference{ + {SourceID: "fake-compiler", ModID: "wolf-mount", Version: "1.0", FileIDs: []string{"exmodz-b"}}, + {SourceID: "fake-compiler", ModID: "bear-mount", Version: "1.0", FileIDs: []string{"exmodz-a"}}, + }) + require.NoError(t, err) + + _, err = svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + after, err := os.ReadFile(deployedPath) + require.NoError(t, err) + require.Equal(t, "BA", string(after), "reordering must be reflected in a subsequent sync (fingerprint changed)") +} +``` + +**`ApplyUpdate`/`ApplyInstall` are exercised indirectly, not with a dedicated new test each** — both already have extensive existing test coverage (`flows_update_test.go`, install-flow test files) that this task's Step 8 (full suite run) must keep green; adding a merged-pak-specific assertion to either would require standing up a much larger fixture (a real update/install flow, not just an enable/disable) for marginal additional confidence beyond what `TestSyncMergedPak_*` (Task 6) and `TestDeployProfile_SyncsMergedPak` (Task 7) already establish for the underlying `syncMergedPak` call itself — the wiring here is mechanically identical to Task 7's, so this task trusts that pattern and spends its test budget on the two NEW code shapes (`ApplyProfileSwitch`'s `plan.To` targeting, `ReorderProfileMods`'s new wrapper) instead. + +- [ ] **Step 2: Run the tests, confirm they fail** + +Run: `go test ./internal/core/... -run 'TestApplyProfileSwitch_SyncsMergedPakForToProfile|TestReorderProfileMods_SyncsMergedPak' -v` +Expected: `TestApplyProfileSwitch_SyncsMergedPakForToProfile` FAILs (no sync wired yet); `TestReorderProfileMods_SyncsMergedPak` fails to COMPILE (`ReorderProfileMods` undefined). + +- [ ] **Step 3: Wire `ApplyProfileSwitch`** + +`internal/core/flows.go`, `ApplyProfileSwitch` ends with `return result, nil` (around line 2644, per this task's own investigation — no existing trailing whole-profile step, unlike `DeployProfile`). Insert immediately before it: + +```go + if syncWarnings, syncErr := s.syncMergedPak(ctx, game, plan.To); syncErr != nil { + result.Notes = append(result.Notes, fmt.Sprintf("Warning: could not sync merged pak: %v", syncErr)) + } else { + for _, w := range syncWarnings { + result.Notes = append(result.Notes, "Warning: "+w) + } + } + + return result, nil +``` + +(`plan.To`, not `plan.From` — the switch deploys INTO `plan.To`, per this function's own doc comment already read during investigation; `SwitchResult.Notes []string` is the pre-existing field this function already uses for its other non-fatal diagnostics.) + +- [ ] **Step 4: Wire `ApplyUpdate`** + +`internal/core/flows.go`, `ApplyUpdate` ends with `return result, nil` (around line 4468). Insert immediately before it: + +```go + if syncWarnings, syncErr := s.syncMergedPak(ctx, game, profileName); syncErr != nil { + result.Warnings = append(result.Warnings, fmt.Sprintf("syncing merged pak: %v", syncErr)) + } else { + result.Warnings = append(result.Warnings, syncWarnings...) + } + + return result, nil +``` + +(`UpdateApplyResult.Warnings []string` — #196's own field, matching how `ApplyRecompile`, Task 9 below, already reports diagnostics.) + +- [ ] **Step 5: Wire `ApplyInstall`** + +`internal/core/flows.go`, `ApplyInstall` ends with `return result, nil` (around line 3650). `ApplyInstall` takes `plan *InstallPlan` and `opts InstallOptions` — confirm `opts.ProfileName` (or `plan`'s own profile field — read `internal/core/flows.go`'s `InstallPlan`/`InstallOptions` struct definitions first, `grep -n "type InstallPlan struct\|type InstallOptions struct" -A 15 internal/core/flows.go`) is the correct profile name to pass; using whichever field the function's OWN body already reads for its per-mod `installer.Install(ctx, game, ..., profileName)` calls keeps this consistent. Insert immediately before the final `return result, nil`: + +```go + if syncWarnings, syncErr := s.syncMergedPak(ctx, game, profileName); syncErr != nil { + result.Warnings = append(result.Warnings, fmt.Sprintf("syncing merged pak: %v", syncErr)) + } else { + result.Warnings = append(result.Warnings, syncWarnings...) + } + + return result, nil +``` + +(`InstallResult.Warnings []string` — confirmed to exist during this task's own investigation, `internal/core/flows.go:3144` area.) + +- [ ] **Step 6: Add `Service.ReorderProfileMods`** + +Add to `internal/core/profile.go` (near `NewProfileManager`/other `Service`-level profile wrappers — if none exist there, add to `flows.go` next to `EnableMod`): + +```go +// ReorderProfileMods persists mods as gameID/profileName's new load order +// (via ProfileManager.ReorderMods) and syncs the merged pak (#197: a +// load-order change is a documented regeneration trigger, since profile +// load order IS merge-application order - see enabledExmodzSources). The +// single seam cmd/lmm and internal/tui both call, replacing their +// previous direct pm.ReorderMods(...) calls (CLI+TUI parity). +// +// A sync failure is non-fatal and returned as part of the SAME error only +// if the reorder itself also failed; a reorder that succeeded but whose +// merged-pak sync failed still returns nil - the reorder took effect, and +// `lmm update`/`lmm verify` are the safety net for a merged pak that +// didn't catch up. Callers wanting to surface a sync warning distinctly +// can call Service.syncMergedPak's own exported test seam directly in a +// follow-up if this proves too quiet in practice; kept simple here to +// match ReorderMods' own existing bare-error signature rather than +// inventing a new result type for one warning slice. +func (s *Service) ReorderProfileMods(gameID, profileName string, mods []domain.ModReference) error { + pm := NewProfileManager(s.configDir, s.db) + if err := pm.ReorderMods(gameID, profileName, mods); err != nil { + return err + } + game, ok := s.games[gameID] + if !ok { + return nil // an unknown game has no merged pak to sync either + } + _, _ = s.syncMergedPak(context.Background(), game, profileName) //nolint:errcheck // best-effort, see doc comment + return nil +} +``` + +Add `"context"` to `profile.go`'s import block if not already present. + +- [ ] **Step 7: Update the CLI and TUI call sites** + +`cmd/lmm/profile.go:831` — change `pm.ReorderMods(game.ID, profileName, newRefs)` to `service.ReorderProfileMods(game.ID, profileName, newRefs)` (read the surrounding ~10 lines first to confirm the local variable name for the `*core.Service` in scope — likely `service`, matching every other command in this package). + +`internal/tui/service_core.go:978` — change `pm.ReorderMods(game.ID, profileName, mods)` to `p.svc.ReorderProfileMods(game.ID, profileName, mods)` (matching this file's own `p.svc` receiver convention used throughout `coreProvider`'s other methods). + +- [ ] **Step 8: Run the tests, confirm they pass** + +Run: `go test ./internal/core/... -run 'TestApplyProfileSwitch_SyncsMergedPakForToProfile|TestReorderProfileMods_SyncsMergedPak' -v` +Expected: both PASS. + +- [ ] **Step 9: Run the full build and suite** + +Run: `go build ./... 2>&1 | tail -60` +Expected: `internal/core/updater.go` remains the only broken file (Task 9). `cmd/lmm` and `internal/tui` build clean (Step 7's call-site swap is a drop-in replacement — `Service.ReorderProfileMods` has the identical `(gameID, profileName string, mods []domain.ModReference) error` signature `pm.ReorderMods` had, just via `service`/`p.svc` instead of a locally-constructed `pm`). + +Run: `go test ./internal/core/... ./cmd/lmm/... ./internal/tui/... 2>&1 | tail -100` +Expected: `internal/core` all green except `updater.go`'s own tests; `cmd/lmm`/`internal/tui` fully green (their `ReorderMods` tests exercise the exact same underlying `pm.ReorderMods` call, now one hop further through `Service`). + +- [ ] **Step 10: Commit** + +```bash +git add internal/core/flows.go internal/core/profile.go internal/core/merged_pak_hooks_test.go cmd/lmm/profile.go internal/tui/service_core.go +git commit -m "feat: sync merged pak on profile switch/update/install/reorder (#197)" +``` + +### Task 9: `CheckGameUpdates` gains `profileName` + merged-pak staleness check + `ApplyMergedPakRegen` + +**Files:** + +- Modify: `internal/core/merged_pak.go` (extract `currentMergedFingerprint`; add `CheckMergedPakStaleness`; add `ApplyMergedPakRegen`) +- Modify: `internal/core/updater.go` (remove `CheckBaseStaleness`, `ApplyRecompile`, `ClassifyRetainedSourceStatError`; change `CheckGameUpdates` signature) +- Modify: `internal/domain/mod.go` (no change — `Update.RecompileNeeded` already exists from #196, reused as-is) +- Test: `internal/core/service_base_staleness_test.go` (rewritten — see Step 6) +- Test: `internal/core/service_apply_recompile_test.go` (deleted — see Step 6) +- Test: `internal/core/updater_test.go` (call-site signature update) + +**Interfaces:** + +- Consumes: `enabledExmodzSources`/`resolveBasePak`/`basePakIndexHash`/`md5File`/`readMergedFingerprint`/`mergedFingerprintsEqual` (Task 5/6). +- Produces: `Service.CheckGameUpdates(ctx context.Context, game *domain.Game, profileName string, installed []domain.InstalledMod) ([]domain.Update, error)` (SIGNATURE CHANGE — `profileName` inserted as the 3rd parameter); `Service.CheckMergedPakStaleness(game *domain.Game, profileName string) (*domain.Update, error)` (nil, nil when not stale or not applicable); `Service.ApplyMergedPakRegen(ctx context.Context, game *domain.Game, profileName string, progress func(DeployProgress)) (*UpdateApplyResult, error)`. + +- [ ] **Step 1: Write the failing tests** + +Delete `internal/core/service_apply_recompile_test.go` and `internal/core/service_base_staleness_test.go` entirely (both test `CheckBaseStaleness`/`ApplyRecompile`, which this task removes — their premise, per-mod fingerprinting, no longer exists). + +Create `internal/core/merged_pak_staleness_test.go`: + +```go +package core_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/stretchr/testify/require" +) + +func TestCheckMergedPakStaleness_NotStaleWhenUnchanged(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + + upd, err := svc.CheckMergedPakStaleness(game, "default") + require.NoError(t, err) + require.Nil(t, upd, "an up-to-date merged pak must not be reported stale") +} + +func TestCheckMergedPakStaleness_StaleAfterModEnable(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "wolf-mount", "1.0", "exmodz-file", []byte("wolf-bytes")) + + upd, err := svc.CheckMergedPakStaleness(game, "default") + require.NoError(t, err) + require.NotNil(t, upd) + require.True(t, upd.RecompileNeeded) + require.Equal(t, upd.InstalledMod.Version, upd.NewVersion, "a staleness row has no real version change") +} + +func TestCheckMergedPakStaleness_NilWhenNoMergedPakEverGenerated(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + upd, err := svc.CheckMergedPakStaleness(game, "default") + require.NoError(t, err) + require.Nil(t, upd, "zero enabled exmodz mods means nothing to report - not an error, not a staleness row") +} + +func TestCheckMergedPakStaleness_NonCompileGame_Nil(t *testing.T) { + svc := newFlowsTestService(t) + game := &domain.Game{ID: "skyrim-se", ModPath: t.TempDir(), DeployMode: domain.DeployExtract} + require.NoError(t, svc.AddGame(game)) + upd, err := svc.CheckMergedPakStaleness(game, "default") + require.NoError(t, err) + require.Nil(t, upd) +} + +// TestApplyMergedPakRegen_Regenerates proves the apply-side wiring: given +// a stale merged pak, applying regenerates and redeploys it. +func TestApplyMergedPakRegen_Regenerates(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "wolf-mount", "1.0", "exmodz-file", []byte("wolf-bytes")) + + result, err := svc.ApplyMergedPakRegen(context.Background(), game, "default", nil) + require.NoError(t, err) + require.NotNil(t, result) + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + data, err := os.ReadFile(deployedPath) + require.NoError(t, err) + require.Equal(t, "bear-byteswolf-bytes", string(data)) +} + +// TestApplyMergedPakRegen_LockedModDiffStillParticipates is the dedicated +// coordinator-flagged design-decision test - see Task 13 for the FULL +// suite; this is the minimal smoke case proving a LOCKED mod's retained +// exmodz is not excluded from a merge triggered by an UNLOCKED mod's +// change. +func TestApplyMergedPakRegen_LockedModDiffStillParticipates(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("locked-bear-bytes")) + pm := svc.NewProfileManager() + require.NoError(t, pm.SetModLock(game.ID, "default", "fake-compiler", "bear-mount", "")) + + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "wolf-mount", "1.0", "exmodz-file", []byte("wolf-bytes")) + + _, err := svc.ApplyMergedPakRegen(context.Background(), game, "default", nil) + require.NoError(t, err, "a locked mod elsewhere in the profile must not block the merge") + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + data, err := os.ReadFile(deployedPath) + require.NoError(t, err) + require.Contains(t, string(data), "locked-bear-bytes", "the locked mod's diff must still be included in the merge") + require.Contains(t, string(data), "wolf-bytes") +} +``` + +Update `internal/core/updater_test.go`'s existing `CheckUpdates`/`CheckGameUpdates` call sites (search `grep -n "CheckGameUpdates(" internal/core/updater_test.go`) to pass a `profileName` argument — every existing test seeds mods under `"default"`, so pass `"default"` as the new 3rd positional argument. + +- [ ] **Step 2: Run the tests, confirm they fail to compile** + +Run: `go test ./internal/core/... -run 'TestCheckMergedPakStaleness|TestApplyMergedPakRegen' -v` +Expected: build failure — `undefined: (*core.Service).CheckMergedPakStaleness`, `undefined: (*core.Service).ApplyMergedPakRegen`. + +- [ ] **Step 3: Extract `currentMergedFingerprint` from `syncMergedPak`** + +In `internal/core/merged_pak.go`, refactor `syncMergedPak`'s fingerprint-building block (everything from `basePakPath, err := resolveBasePak(game)` through the `current.Mods[i].Version = ...` loop) into a new shared function, so `CheckMergedPakStaleness` (Step 4) doesn't duplicate it: + +```go +// currentMergedFingerprint computes what game+profileName's merged pak +// SHOULD look like right now: the live base pak's IndexHash plus every +// currently-enabled exmodz mod's identity/version/content checksum, in +// profile load order. Returns a nil sources/zero-value fingerprint (not an +// error) when there is nothing to merge - callers distinguish "nothing to +// do" from "failed to compute" via the returned slice's length, exactly +// like syncMergedPak's own zero-sources branch does. +func (s *Service) currentMergedFingerprint(game *domain.Game, profileName string) (MergedFingerprint, []source.MergeSource, error) { + sources, err := s.enabledExmodzSources(game, profileName) + if err != nil { + return MergedFingerprint{}, nil, fmt.Errorf("listing enabled exmodz mods: %w", err) + } + if len(sources) == 0 { + return MergedFingerprint{}, sources, nil + } + + basePakPath, err := resolveBasePak(game) + if err != nil { + return MergedFingerprint{}, sources, err + } + liveHash, err := basePakIndexHash(basePakPath) + if err != nil { + return MergedFingerprint{}, sources, fmt.Errorf("reading base pak for merge fingerprint: %w", err) + } + + current := MergedFingerprint{BaseIndexHash: liveHash} + for _, src := range sources { + sum, herr := md5File(src.ExmodzPath) + if herr != nil { + return MergedFingerprint{}, sources, fmt.Errorf("hashing %s: %w", src.ExmodzPath, herr) + } + sourceID, modID, _ := strings.Cut(src.ModRef, ":") + current.Mods = append(current.Mods, MergedFingerprintEntry{SourceID: sourceID, ModID: modID, Checksum: sum}) + } + + mods, err := s.GetInstalledModsInProfileOrder(game.ID, profileName) + if err != nil { + return MergedFingerprint{}, sources, fmt.Errorf("loading profile mods: %w", err) + } + versionByRef := make(map[string]string, len(mods)) + for _, m := range mods { + versionByRef[m.SourceID+":"+m.ID] = m.Version + } + for i, src := range sources { + current.Mods[i].Version = versionByRef[src.ModRef] + } + + return current, sources, nil +} +``` + +Now simplify `syncMergedPak` (Task 6) to call this instead of repeating the block — replace everything from `basePakPath, err := resolveBasePak(game)` through the `current.Mods[i].Version = ...` loop with: + +```go + current, sources, err := s.currentMergedFingerprint(game, profileName) + if err != nil { + return nil, err + } +``` + +(`sources` here SHADOWS the outer `sources` variable `syncMergedPak` already computed via its own earlier `enabledExmodzSources` call for the zero-check — since `currentMergedFingerprint` recomputes it internally anyway, DELETE `syncMergedPak`'s own earlier `sources, err := s.enabledExmodzSources(...)` call and its zero-length check, replacing BOTH with a single call to `currentMergedFingerprint` right after the `game.DeployMode != domain.DeployCompile` guard, THEN branch on `len(sources) == 0` for the uninstall-to-zero path. Re-read the resulting full function once assembled to confirm there is exactly ONE call to `enabledExmodzSources`, indirectly via `currentMergedFingerprint`, not two.) `basePakPath` is still needed later (the `mc.MergeCompile(ctx, basePakPath, ...)` call) — keep a separate `basePakPath, err := resolveBasePak(game)` call in `syncMergedPak` after the zero-check (cheap — it's just an `os.Stat`, unlike `basePakIndexHash` which `currentMergedFingerprint` already paid for). + +- [ ] **Step 4: Implement `CheckMergedPakStaleness`** + +Append to `internal/core/merged_pak.go`: + +```go +// CheckMergedPakStaleness reports whether game+profileName's merged pak no +// longer matches the current enabled-mod set/order/versions/base pak +// (#197, generalizing #196's per-mod CheckBaseStaleness to the merged +// model). Returns nil, nil - not an error - when the merged pak is +// up to date, when there is nothing to merge (zero enabled exmodz mods), +// or when game is not a DeployCompile game. +func (s *Service) CheckMergedPakStaleness(game *domain.Game, profileName string) (*domain.Update, error) { + if game.DeployMode != domain.DeployCompile { + return nil, nil + } + + current, sources, err := s.currentMergedFingerprint(game, profileName) + if err != nil { + return nil, err + } + if len(sources) == 0 { + return nil, nil + } + + gameCache := s.GetGameCache(game) + cachePath := gameCache.ModPath(game.ID, domain.SourceMerged, mergedPakModID, mergedPakVersion) + stored, ok := readMergedFingerprint(cachePath) + if ok { + if eq, eqErr := mergedFingerprintsEqual(current, stored); eqErr == nil && eq { + return nil, nil + } + } + + return &domain.Update{ + InstalledMod: domain.InstalledMod{ + Mod: domain.Mod{ + ID: mergedPakModID, SourceID: domain.SourceMerged, + Name: "Icarus Merged Pak", Version: mergedPakVersion, GameID: game.ID, + }, + }, + NewVersion: mergedPakVersion, + RecompileNeeded: true, + }, nil +} +``` + +- [ ] **Step 5: Implement `ApplyMergedPakRegen`** + +Append to `internal/core/merged_pak.go`: + +```go +// ApplyMergedPakRegen regenerates game+profileName's merged pak (#197 - +// replaces #196's per-mod ApplyRecompile). No lock gate: a locked mod's +// retained exmodz still participates in every re-merge (design decision 3 +// - locking pins THAT mod's own version, it does not freeze the whole +// merged pak or exclude the mod's diff; reading a locked mod's retained +// source to feed the merge is not "touching" it in the sense a lock +// protects against). +func (s *Service) ApplyMergedPakRegen(ctx context.Context, game *domain.Game, profileName string, progress func(DeployProgress)) (*UpdateApplyResult, error) { + result := &UpdateApplyResult{} + warnings, err := s.syncMergedPak(ctx, game, profileName) + if err != nil { + return result, err + } + result.Warnings = warnings + result.Applied = []string{mergedPakFileName} + if progress != nil { + progress(DeployProgress{Phase: UpdateDownloadDone}) + } + return result, nil +} +``` + +- [ ] **Step 6: Remove `CheckBaseStaleness`/`ApplyRecompile`/`ClassifyRetainedSourceStatError` from `updater.go`** + +Delete these three functions and their doc comments in full from `internal/core/updater.go` (search `grep -n "^func (s \*Service) CheckBaseStaleness\|^func ClassifyRetainedSourceStatError\|^func (s \*Service) ApplyRecompile" internal/core/updater.go` for exact current line ranges — each runs to its own closing `}` before the next function/EOF). Remove now-unused imports this leaves behind (`"io/fs"` was added in #196's review-fix round specifically for `ClassifyRetainedSourceStatError` — check `grep -n '"io/fs"' internal/core/updater.go` and remove it if nothing else in this file uses `fs.` anymore). + +- [ ] **Step 7: Change `CheckGameUpdates`'s signature** + +`internal/core/updater.go`'s `CheckGameUpdates` — add `profileName string` as the 3rd parameter and replace its `CheckBaseStaleness` call with `CheckMergedPakStaleness`: + +```go +func (s *Service) CheckGameUpdates(ctx context.Context, game *domain.Game, profileName string, installed []domain.InstalledMod) ([]domain.Update, error) { + updates, checkErr := s.NewUpdater().CheckUpdates(ctx, game, installed) + + staleUpd, staleErr := s.CheckMergedPakStaleness(game, profileName) + if staleErr != nil && checkErr == nil { + checkErr = staleErr + } + + if staleUpd != nil { + reported := false + for _, u := range updates { + if u.InstalledMod.SourceID == staleUpd.InstalledMod.SourceID && u.InstalledMod.ID == staleUpd.InstalledMod.ID { + reported = true + break + } + } + if !reported { + updates = append(updates, *staleUpd) + } + } + + return updates, checkErr +} +``` + +(The "already reported" de-dup check that #196's version needed — a mod with BOTH a real update and staleness only reporting the real update — is now moot for the SAME reason it moots itself here too: `staleUpd`'s identity is always the SYNTHETIC merged-pak row, which by construction can never collide with a REAL installed mod's `(SourceID, ID)` pair from `updates`, so the loop above is a defensive no-op today, not load-bearing — kept for clarity/future-proofing rather than removed, since a future change adding a second staleness source could reintroduce exactly this collision.) + +- [ ] **Step 8: Run the tests, confirm they pass** + +Run: `go test ./internal/core/... -run 'TestCheckMergedPakStaleness|TestApplyMergedPakRegen' -v` +Expected: all 7 PASS, including the locked-mod smoke test. + +- [ ] **Step 9: Run the full build and suite** + +Run: `go build ./... 2>&1 | tail -80` +Expected: `internal/core` builds clean. `cmd/lmm` and `internal/tui` now fail to build (their `CheckGameUpdates(ctx, game, installed)` call sites are missing the new `profileName` argument) — fixed in Task 10/12. + +Run: `go test ./internal/core/... 2>&1 | tail -100` +Expected: all green. + +- [ ] **Step 10: Commit** + +```bash +git add internal/core/merged_pak.go internal/core/updater.go internal/core/updater_test.go internal/core/merged_pak_staleness_test.go +git rm internal/core/service_apply_recompile_test.go internal/core/service_base_staleness_test.go +git commit -m "feat: CheckMergedPakStaleness + ApplyMergedPakRegen, retire per-mod #196 staleness (#197)" +``` + +### Task 10: CLI wiring — `cmd/lmm/update.go` + +**Files:** + +- Modify: `cmd/lmm/update.go` (both `CheckGameUpdates` call sites; `applyRecompile` → calls `ApplyMergedPakRegen`; rendering text unchanged — it already speaks generically about "recompile"/"base pak updated," which reads correctly for the merged pak too) +- Test: `cmd/lmm/update_recompile_test.go` (rewritten fixture — see Step 1) + +**Interfaces:** + +- Consumes: `Service.CheckGameUpdates(ctx, game, profileName, installed)` (Task 9, new signature); `Service.ApplyMergedPakRegen` (Task 9). +- Produces: nothing new — the existing `updateModJSON.RecompileNeeded`/`Reason` fields (#196) and `singleUpdateJSON` `"recompiled"`/`"recompile_available"` statuses (#196) are REUSED as-is for the merged-pak row; no JSON contract change. + +The bulk table's `"[recompile]"` POLICY marker, the single-mod `"Recompiling %s (base pak updated)..."` text, and the `--json` `recompile_needed`/`reason: "stale_compile"` fields were all written generically in #196 (they never say "this specific mod" in a way that stops making sense for a profile-level row) — this task changes ONLY the two `CheckGameUpdates` call sites' argument list and the apply-dispatch target function name; no rendering code changes. + +- [ ] **Step 1: Write the failing test** + +Rewrite `cmd/lmm/update_recompile_test.go`'s `setupDoUpdateRecompileTest` helper (the shared fixture for all 4 existing tests in that file) to seed a MERGED-PAK-eligible mod instead of a #196-era per-mod-compiled one: + +```go +// setupDoUpdateRecompileTest builds a DeployCompile game with a registered +// merge-compiler-capable source and an ENABLED exmodz mod, deliberately +// leaving the merged pak un-generated (or stale, per staleAfterSync) so +// `lmm update` reports/applies a #197 merge-needed row end to end through +// the CLI. linkMethod is LinkCopy so a successful regen+redeploy is +// provable from the on-disk deployed bytes (a symlink would trivially +// reflect an in-place cache swap on its own). +func setupDoUpdateRecompileTest(t *testing.T) (*core.Service, *domain.Game, *compilerInstallSource, string) { + t.Helper() + + configDir = t.TempDir() + dataDir = t.TempDir() + + installDir := t.TempDir() + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + writeFakeBasePak(t, basePak) + + svc, err := core.NewService(core.ServiceConfig{ConfigDir: configDir, DataDir: dataDir, CacheDir: t.TempDir()}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + compiler := &compilerInstallSource{fakeInstallSource: newFakeInstallSource("fake-compiler")} + svc.RegisterSource(compiler) + + game := &domain.Game{ + ID: "icarus", Name: "Icarus", InstallPath: installDir, ModPath: t.TempDir(), + DeployMode: domain.DeployCompile, LinkMethod: domain.LinkCopy, + SourceIDs: map[string]string{"fake-compiler": "external-icarus-id"}, + } + require.NoError(t, svc.AddGame(game)) + + oldSource, oldProfile, oldAll, oldDryRun, oldForce := updateSource, updateProfile, updateAll, updateDryRun, updateForce + oldVerbose, oldNoColor, oldNoHooks := verbose, noColor, noHooks + updateSource = "fake-compiler" + updateProfile = "" + updateAll = false + updateDryRun = false + updateForce = false + verbose = false + noColor = true + noHooks = false + t.Cleanup(func() { + updateSource, updateProfile, updateAll, updateDryRun, updateForce = oldSource, oldProfile, oldAll, oldDryRun, oldForce + verbose, noColor, noHooks = oldVerbose, oldNoColor, oldNoHooks + }) + + const modID, version, fileID = "bear-mount", "3.3", "exmodz-file-id" + gameCache := svc.GetGameCache(game) + require.NoError(t, gameCache.Store(game.ID, "fake-compiler", modID, version, cache.RetainedSourceName(fileID), []byte("retained-exmodz-bytes"))) + + im := &domain.InstalledMod{ + Mod: domain.Mod{ID: modID, SourceID: "fake-compiler", Name: "Bear Mount", Version: version, GameID: game.ID}, + ProfileName: "default", + UpdatePolicy: domain.UpdateNotify, + Enabled: true, + FileIDs: []string{fileID}, + } + require.NoError(t, svc.SaveInstalledMod(im)) + + pm := svc.NewProfileManager() + _, cerr := pm.Create(game.ID, "default") + require.NoError(t, cerr) + require.NoError(t, pm.UpsertMod(game.ID, "default", domain.ModReference{SourceID: "fake-compiler", ModID: modID, Version: version, FileIDs: []string{fileID}})) + + return svc, game, compiler, filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") +} +``` + +`compilerInstallSource` (defined in `cmd/lmm/install_compile_test.go`, #173/#196-era) needs its `Compile` method replaced with `ValidateSource`/`MergeCompile`, matching Task 2 Step 1's `fakeCompilerSource` change exactly — apply the identical transformation there too (same struct shape: `*fakeInstallSource` embedded, `compileCalls`/`validateCalls int` fields, `var _ source.MergeCompiler = (*compilerInstallSource)(nil)`). + +The 4 existing tests in `cmd/lmm/update_recompile_test.go` (`TestDoUpdate_JSON_ReportsRecompileNeeded`, `TestApplySingleUpdate_Recompile_AppliesAndRedeploys`, `TestApplySingleUpdate_Recompile_JSON`, `TestApplySingleUpdate_Recompile_LockedRefuses`) need NO other changes — they already only reference `row.RecompileNeeded`/`row.Reason`/deployed-file content, which stay semantically valid (now describing the merged pak instead of a per-mod compile). **Exception:** `TestApplySingleUpdate_Recompile_LockedRefuses` currently locks the SAME mod whose staleness row it expects to be refused — under #197, the merged pak is a SEPARATE synthetic mod (`sourceID="lmm-merged"`, `modID="merged-pak"`), and design decision 3 says a locked CONTRIBUTING mod does NOT block the merge. Delete this test — the correct #197 replacement (proving a lock does NOT block, the opposite assertion) lives in Task 13's dedicated locked-mod suite; keeping a test here that asserts the OLD, now-wrong behavior would be actively misleading. + +- [ ] **Step 2: Run the tests, confirm they fail** + +Run: `go test ./cmd/lmm/... -run 'TestDoUpdate_JSON_ReportsRecompileNeeded|TestApplySingleUpdate_Recompile' -v` +Expected: build failure (`compilerInstallSource` doesn't implement `source.MergeCompiler` until Step 1's transformation is applied there too) and/or `service.CheckGameUpdates` argument-count mismatch until Step 3 lands. + +- [ ] **Step 3: Update the two `CheckGameUpdates` call sites** + +`cmd/lmm/update.go:319`: + +```go + updates, checkErr := service.CheckGameUpdates(ctx, game, profileName, installed) +``` + +`cmd/lmm/update.go:601`: + +```go + updates, err := service.CheckGameUpdates(ctx, game, profileName, []domain.InstalledMod{*mod}) +``` + +(`profileName` is already in scope at both call sites — `doUpdate`'s own resolved profile, and `applySingleUpdate`'s parameter of the same name, respectively; confirm by reading the ~15 lines above each call site before editing.) + +- [ ] **Step 4: Retarget `applyRecompile`** + +`cmd/lmm/update.go`'s `applyRecompile` function — change its final line from `service.ApplyRecompile(ctx, game, profileName, mod, progress)` to `service.ApplyMergedPakRegen(ctx, game, profileName, progress)`, and drop the now-unused `mod domain.InstalledMod` parameter (it was only ever passed through to `ApplyRecompile`, which no longer exists): + +```go +// applyRecompile applies a #197 merged-pak staleness row via +// Service.ApplyMergedPakRegen, printing from its progress events the same +// way applyUpdate does for its own (UpdateWarning/UpdateNote are the only +// phases ApplyMergedPakRegen emits - it runs no hooks and downloads +// nothing worth a progress bar). +func applyRecompile(ctx context.Context, service *core.Service, game *domain.Game, profileName string) error { + progress := func(p core.DeployProgress) { + switch p.Phase { + case core.UpdateWarning: + fmt.Fprintf(os.Stderr, "Warning: %s\n", p.Detail) + case core.UpdateNote: + if verbose && !jsonOutput { + fmt.Printf(" %s\n", p.Detail) + } + } + } + + _, err := service.ApplyMergedPakRegen(ctx, game, profileName, progress) + return err +} +``` + +Update `applyUpdate`'s own call site (`return applyRecompile(ctx, service, game, upd.InstalledMod, profileName)`) to drop the now-removed argument: `return applyRecompile(ctx, service, game, profileName)`. + +- [ ] **Step 5: Run the tests, confirm they pass** + +Run: `go test ./cmd/lmm/... -run 'TestDoUpdate_JSON_ReportsRecompileNeeded|TestApplySingleUpdate_Recompile' -v` +Expected: all 3 remaining tests PASS (the 4th, `LockedRefuses`, was deleted in Step 1). + +- [ ] **Step 6: Run the full `cmd/lmm` suite** + +Run: `go test ./cmd/lmm/... 2>&1 | tail -80` +Expected: green. `go build ./... 2>&1 | tail -40` — `internal/tui` remains the only broken package (Task 12). + +- [ ] **Step 7: Commit** + +```bash +git add cmd/lmm/update.go cmd/lmm/update_recompile_test.go cmd/lmm/install_compile_test.go +git commit -m "feat: cmd/lmm/update.go targets merged-pak regen instead of per-mod recompile (#197)" +``` + +### Task 11: CLI wiring — `cmd/lmm/verify.go` + +**Files:** + +- Modify: `cmd/lmm/verify.go:308-340` (replace the per-mod `CheckBaseStaleness` pre-pass with a profile-level `CheckMergedPakStaleness` check) +- Test: `cmd/lmm/verify_recompile_test.go` (rewritten fixture) + +**Interfaces:** + +- Consumes: `Service.CheckMergedPakStaleness(game, profile)` (Task 9). +- Produces: nothing new — `verifyFileJSON.Status == "stale_compile"` (#196) is REUSED for the merged-pak row. + +- [ ] **Step 1: Write the failing test** + +Rewrite `cmd/lmm/verify_recompile_test.go`'s two tests to reuse Task 10's rewritten `setupDoUpdateRecompileTest` fixture (same file convention already established — `verify_recompile_test.go` already calls into `update_recompile_test.go`'s helper today, per its own existing structure) with NO source changes needed to the test bodies themselves — `TestDoVerify_StaleCompile_ReportedAsWarning` and `TestDoVerify_StaleCompile_JSON` already just assert on the `"RECOMPILE NEEDED"`/`"stale_compile"` text and `verifyFileJSON` shape, which is unchanged. Confirm by reading `cmd/lmm/verify_recompile_test.go` in full before touching `verify.go` — if it compiles and passes unmodified once Task 10's fixture change lands, skip straight to Step 3; if `svc.SaveFileChecksum("fake-compiler", "bear-mount", game.ID, "default", "exmodz-file-id", "deadbeef")` (a line in the existing test, seeding a checksum so `doVerify`'s OTHER pre-existing checks stay quiet) still makes sense against Task 10's rewritten fixture's exact fileID/modID naming, no change is needed there either. + +- [ ] **Step 2: Run the tests, confirm they still describe the intended behavior** + +Run: `go test ./cmd/lmm/... -run 'TestDoVerify_StaleCompile' -v` +Expected (before Step 3's `verify.go` change lands): these tests currently call into `doVerify`, which still calls the now-removed `svc.CheckBaseStaleness` — build failure. This confirms Step 3 is required, not skippable. + +- [ ] **Step 3: Replace the staleness pre-pass** + +`cmd/lmm/verify.go:308-340` (quoted above) — replace the whole `if game.DeployMode == domain.DeployCompile { ... }` block with: + +```go + // Merged-pak staleness check (#197, generalizing #196's per-mod + // version): for a DeployCompile game, compare the profile's merged + // pak's recorded fingerprint against the game's CURRENT enabled-mod + // set/order/versions/base pak. Entirely local/offline. modFilter has no + // effect here - the merged pak is profile-scoped, not per-mod, so + // `lmm verify ` still checks it (a single mod's own version + // mismatch and the profile's overall merge staleness are independent + // facts). + if game.DeployMode == domain.DeployCompile { + staleUpd, serr := svc.CheckMergedPakStaleness(game, profile) + if serr != nil { + if jsonOutput { + jsonFiles = append(jsonFiles, verifyFileJSON{Status: "skipped", Note: fmt.Sprintf("could not check merged pak staleness: %v", serr)}) + } else { + fmt.Printf("%s could not check merged pak staleness: %v\n", colorYellow("?"), serr) + } + warnings++ + } + checked++ + if staleUpd != nil { + if jsonOutput { + jsonFiles = append(jsonFiles, verifyFileJSON{ModID: staleUpd.InstalledMod.ID, ModName: staleUpd.InstalledMod.Name, Status: "stale_compile"}) + } else { + fmt.Printf("%s %s - RECOMPILE NEEDED (base pak updated - run 'lmm update' to fix)\n", colorYellow("?"), staleUpd.InstalledMod.Name) + } + warnings++ + } + } +``` + +`modFilter`/`installedMods` are no longer read by this block (the check is profile-scoped, not per-mod) — this is intentional per the doc comment above, not a bug; do not restore the old per-mod filtering loop. + +- [ ] **Step 4: Run the tests, confirm they pass** + +Run: `go test ./cmd/lmm/... -run 'TestDoVerify_StaleCompile' -v` +Expected: both PASS. + +- [ ] **Step 5: Run the full `cmd/lmm` suite** + +Run: `go test ./cmd/lmm/... 2>&1 | tail -80` +Expected: green. + +- [ ] **Step 6: Commit** + +```bash +git add cmd/lmm/verify.go cmd/lmm/verify_recompile_test.go +git commit -m "feat: lmm verify checks merged-pak staleness at the profile level (#197)" +``` + +### Task 12: TUI wiring — `internal/tui/service_core.go` + +**Files:** + +- Modify: `internal/tui/service_core.go` (`coreProvider.CheckUpdates`, `coreProvider.ApplyUpdate`) +- Test: `internal/tui/service_core_recompile_test.go` (rewritten fixture) + +**Interfaces:** + +- Consumes: `Service.CheckGameUpdates(ctx, game, profileName, installed)` (Task 9, new signature); `Service.ApplyMergedPakRegen` (Task 9). +- Produces: nothing new — `UpdateItem.RecompileNeeded`/`VersionLabel()` (#196) are REUSED as-is. + +**Real defect caught while planning this task:** #196's `coreProvider.ApplyUpdate` calls `p.svc.GetInstalledMod(u.Source, u.ID, game.ID, profile)` FIRST, unconditionally, to look up the real `InstalledMod` behind `u` — then re-checks via `CheckGameUpdates` for just that one mod. Under #197 a `RecompileNeeded` row's `u.Source`/`u.ID` are the SYNTHETIC merged-pak identity (`domain.SourceMerged`/`mergedPakModID`), which has NO real `InstalledMod` DB row — `GetInstalledMod` would return an error and abort the whole apply BEFORE ever reaching the dispatch that would have routed it correctly. Fixed below by checking `u.RecompileNeeded` FIRST (already known — `UpdateItem` carries it from the last `CheckUpdates` call, no re-check needed) and short-circuiting straight to `ApplyMergedPakRegen`, mirroring Task 10's identical simplification of `cmd/lmm`'s own `applyRecompile` (which also stopped needing a `mod` parameter). + +- [ ] **Step 1: Write the failing test** + +Rewrite `internal/tui/service_core_recompile_test.go`'s `newRecompileActionsFixture` to seed a merged-pak-eligible mod (mirroring Task 10's `setupDoUpdateRecompileTest` rewrite exactly — same fixture shape, TUI-layer construction): + +```go +func newRecompileActionsFixture(t *testing.T) (tui.ActionProvider, *recompileFakeSource, string) { + t.Helper() + + installDir := t.TempDir() + basePak := filepath.Join(installDir, "Icarus", "Content", "Data", "data.pak") + require.NoError(t, os.MkdirAll(filepath.Dir(basePak), 0o755)) + w, err := unrealpak.Create(basePak) + require.NoError(t, err) + require.NoError(t, w.AddFile("Data/D_Fixture.json", []byte(`{"fixture":true}`))) + require.NoError(t, w.Close()) + + svc, err := core.NewService(core.ServiceConfig{ConfigDir: t.TempDir(), DataDir: t.TempDir(), CacheDir: t.TempDir()}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, svc.Close()) }) + + compiler := &recompileFakeSource{} + svc.RegisterSource(compiler) + + game := &domain.Game{ + ID: "icarus", Name: "Icarus", InstallPath: installDir, ModPath: t.TempDir(), + DeployMode: domain.DeployCompile, LinkMethod: domain.LinkCopy, + SourceIDs: map[string]string{"fake-compiler": "external-icarus-id"}, + } + require.NoError(t, svc.AddGame(game)) + + pm := svc.NewProfileManager() + _, err = pm.Create(game.ID, "default") + require.NoError(t, err) + require.NoError(t, pm.SetDefault(game.ID, "default")) + + const modID, version, fileID = "bear-mount", "3.3", "exmodz-file-id" + gameCache := svc.GetGameCache(game) + require.NoError(t, gameCache.Store(game.ID, "fake-compiler", modID, version, cache.RetainedSourceName(fileID), []byte("retained-exmodz-bytes"))) + + im := &domain.InstalledMod{ + Mod: domain.Mod{ID: modID, SourceID: "fake-compiler", Name: "Bear Mount", Version: version, GameID: game.ID}, + ProfileName: "default", + UpdatePolicy: domain.UpdateNotify, + Enabled: true, + FileIDs: []string{fileID}, + } + require.NoError(t, svc.SaveInstalledMod(im)) + require.NoError(t, pm.UpsertMod(game.ID, "default", domain.ModReference{SourceID: "fake-compiler", ModID: modID, Version: version, FileIDs: []string{fileID}})) + + return tui.NewCoreActions(svc, game, "default"), compiler, filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") +} +``` + +`recompileFakeSource` (defined in this same test file) needs its `Compile` method replaced with `ValidateSource`/`MergeCompile`, matching Task 2 Step 1's transformation exactly (same pattern, third occurrence of this identical fake-rewrite across `internal/core`/`cmd/lmm`/`internal/tui`'s own compiler fakes). The two existing tests in this file (`TestCoreProviderActions_CheckUpdates_ReportsRecompileNeeded`, `TestCoreProviderActions_ApplyUpdate_Recompile_AppliesAndRedeploys`) need no other changes — they already only assert on `u.RecompileNeeded`/`u.VersionLabel()`/deployed-file content. + +- [ ] **Step 2: Run the tests, confirm they fail** + +Run: `go test ./internal/tui/... -run 'TestCoreProviderActions_CheckUpdates_ReportsRecompileNeeded|TestCoreProviderActions_ApplyUpdate_Recompile' -v` +Expected: build failure (`recompileFakeSource` doesn't implement `source.MergeCompiler`; `p.svc.CheckGameUpdates` argument-count mismatch). + +- [ ] **Step 3: Update `CheckUpdates`** + +`internal/tui/service_core.go`'s `CheckUpdates` — change the `CheckGameUpdates` call: + +```go + updates, checkErr := p.svc.CheckGameUpdates(ctx, game, profile, installed) +``` + +(`profile` is already in scope — `p.currentProfile()`'s result, assigned two lines above the existing call.) + +- [ ] **Step 4: Fix `ApplyUpdate`'s dispatch order** + +`internal/tui/service_core.go`'s `ApplyUpdate` — replace the WHOLE function body with: + +```go +func (p *coreProvider) ApplyUpdate(ctx context.Context, u UpdateItem, progress func(ActionProgress)) (ActionOutcome, error) { + game := p.currentGame() + profile := p.currentProfile() + + adapter := deployProgressAdapter(progress, func(p core.DeployProgress) (ActionProgress, bool) { + return updateProgressLine(u.Name, p) + }) + + // #197: a RecompileNeeded row's Source/ID are the SYNTHETIC merged-pak + // identity (domain.SourceMerged/"merged-pak"), which has no real + // InstalledMod DB row - GetInstalledMod below would error for it. u + // already carries everything needed (RecompileNeeded is set by the + // last CheckUpdates call), so this branches BEFORE the GetInstalledMod/ + // re-check path that only makes sense for a real installed mod. + if u.RecompileNeeded { + result, err := p.svc.ApplyMergedPakRegen(ctx, game, profile, adapter) + if err != nil { + return ActionOutcome{}, mapUpdateNetworkError(fmt.Sprintf("regenerating merged pak for %s", u.Name), u.Source, err) + } + return ActionOutcome{ + Message: fmt.Sprintf("Regenerated %q (base pak or mod set updated)", u.Name), + Warnings: mergeDiagnostics(result.Warnings, result.Notes), + }, nil + } + + mod, err := p.svc.GetInstalledMod(u.Source, u.ID, game.ID, profile) + if err != nil { + return ActionOutcome{}, fmt.Errorf("getting installed mod %s: %w", u.Name, err) + } + + updates, err := p.svc.CheckGameUpdates(ctx, game, profile, []domain.InstalledMod{*mod}) + if err != nil { + return ActionOutcome{}, mapUpdateNetworkError(fmt.Sprintf("checking update for %s", u.Name), u.Source, err) + } + if len(updates) == 0 { + return ActionOutcome{Message: notCheckedMessage(u.Name, *mod)}, nil + } + upd := updates[0] + + opts := core.UpdateOptions{ + Hooks: p.resolvedHooks(game, profile), + HookRunner: p.hookRunner(), + HookContext: p.hookContext(game), + Force: false, + } + + result, err := p.svc.ApplyUpdate(ctx, game, profile, upd, opts, adapter) + if err != nil { + return ActionOutcome{}, mapUpdateNetworkError(fmt.Sprintf("updating %s", u.Name), u.Source, err) + } + return ActionOutcome{ + Message: fmt.Sprintf("Updated %q to %s", u.Name, upd.NewVersion), + Warnings: mergeDiagnostics(result.Warnings, result.Notes), + }, nil +} +``` + +(This is the pre-existing function's REAL-update path, verbatim, just moved after the new early `RecompileNeeded` branch instead of running a doomed `GetInstalledMod` call first — `upd.RecompileNeeded`'s OLD re-check-based dispatch, further down in the pre-#197 body, is now unreachable dead code once the early branch exists, since a REAL update's `updates[0]` from `CheckGameUpdates` is never itself a synthetic merged-pak row when `u.RecompileNeeded` was already false going in; delete the old inner `if upd.RecompileNeeded { ... }` block entirely rather than leaving unreachable code behind.) + +- [ ] **Step 5: Run the tests, confirm they pass** + +Run: `go test ./internal/tui/... -run 'TestCoreProviderActions_CheckUpdates_ReportsRecompileNeeded|TestCoreProviderActions_ApplyUpdate_Recompile' -v` +Expected: both PASS. + +- [ ] **Step 6: Run the full `internal/tui` suite, then the full repo** + +Run: `go test ./internal/tui/... 2>&1 | tail -100` +Expected: green — in particular, confirm every PRE-EXISTING `TestCoreProviderActions_ApplyUpdate_*` test (a REAL version update, `RecompileNeeded` false) still passes unchanged through the reordered function; the early branch must be a true no-op for them. + +Run: `go build ./... && go vet ./... && gofmt -l . && go test ./... 2>&1 | tail -60` +Expected: the ENTIRE repo builds and passes now — this is the first point since Task 2 where every package is simultaneously green. + +- [ ] **Step 7: Commit** + +```bash +git add internal/tui/service_core.go internal/tui/service_core_recompile_test.go +git commit -m "feat: TUI targets merged-pak regen instead of per-mod recompile (#197)" +``` + +### Task 13: Locked-mod semantics — dedicated test suite + +**Files:** + +- Test: `internal/core/merged_pak_locked_test.go` (new) + +**Interfaces:** + +- Consumes: `Service.syncMergedPak`, `Service.CheckMergedPakStaleness`, `Service.ApplyMergedPakRegen` (Tasks 6/9); `ProfileManager.SetModLock` (existing, unchanged). +- Produces: nothing new — this task is proof, not implementation. **This is the design decision flagged for coordinator confirmation (plan header, Design Decisions item 3).** If the coordinator reverses the decision (a lock SHOULD freeze the whole merge, or exclude the locked mod's diff), every test in this file gets its assertion inverted and `ApplyMergedPakRegen`/`syncMergedPak` gain a lock-gate check mirroring #196's `ApplyRecompile`'s `ErrModLocked` pattern — a small, contained change, which is exactly why this is broken out as its own task rather than folded into Task 6/9. + +- [ ] **Step 1: Write the tests** + +Create `internal/core/merged_pak_locked_test.go`: + +```go +package core_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestLockedMod_DiffStillParticipatesInMerge: a locked mod's retained +// exmodz contributes to the merge exactly like an unlocked one - locking +// pins THAT mod's own VERSION, it does not exclude its diff or freeze the +// merged pak (design decision 3, flagged for coordinator confirmation). +func TestLockedMod_DiffStillParticipatesInMerge(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + pm := svc.NewProfileManager() + require.NoError(t, pm.SetModLock(game.ID, "default", "fake-compiler", "bear-mount", "")) + + warnings, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err, "a locked mod must not block the merge") + require.Empty(t, warnings) + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + data, err := os.ReadFile(deployedPath) + require.NoError(t, err) + require.Equal(t, "bear-bytes", string(data), "the locked mod's own diff must be included") +} + +// TestLockedMod_DoesNotBlockAnotherModsChangeFromReachingTheMerge: enabling +// a SECOND, unlocked mod alongside a locked one must still trigger +// regeneration and include BOTH mods' diffs. +func TestLockedMod_DoesNotBlockAnotherModsChangeFromReachingTheMerge(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + pm := svc.NewProfileManager() + require.NoError(t, pm.SetModLock(game.ID, "default", "fake-compiler", "bear-mount", "")) + _, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err) + + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "wolf-mount", "1.0", "exmodz-file", []byte("wolf-bytes")) + + warnings, err := svc.SyncMergedPakForTest(context.Background(), game, "default") + require.NoError(t, err, "a lock on one mod must never block ANOTHER mod's change from reaching the merged pak") + require.Empty(t, warnings) + + deployedPath := filepath.Join(game.ModPath, "zzz_LMM_Merged_P.pak") + data, err := os.ReadFile(deployedPath) + require.NoError(t, err) + require.Equal(t, "bear-byteswolf-bytes", string(data), "both mods' diffs must be present - the lock excluded neither") +} + +// TestLockedMod_CheckMergedPakStaleness_NotBlockedByLock proves the CHECK +// side (not just apply) also treats a locked mod normally. +func TestLockedMod_CheckMergedPakStaleness_NotBlockedByLock(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + pm := svc.NewProfileManager() + require.NoError(t, pm.SetModLock(game.ID, "default", "fake-compiler", "bear-mount", "")) + + upd, err := svc.CheckMergedPakStaleness(game, "default") + require.NoError(t, err) + require.NotNil(t, upd, "a never-yet-generated merged pak is stale regardless of a lock elsewhere in the profile") + + _, err = svc.ApplyMergedPakRegen(context.Background(), game, "default", nil) + require.NoError(t, err) + + upd, err = svc.CheckMergedPakStaleness(game, "default") + require.NoError(t, err) + require.Nil(t, upd, "after applying, the locked mod's presence must not cause a spurious permanent-stale state") +} + +// TestLockedMod_ApplyMergedPakRegen_NeverErrorsForALock proves +// ApplyMergedPakRegen has NO lock-gate at all (unlike #196's ApplyRecompile, +// which refused a locked MOD's own recompile) - it is a profile-level +// operation, and design decision 3 explicitly rejects "freeze the whole +// merge on any lock present." +func TestLockedMod_ApplyMergedPakRegen_NeverErrorsForALock(t *testing.T) { + svc, game, _ := newMergedPakTestGame(t) + seedEnabledExmodzMod(t, svc, game, "fake-compiler", "bear-mount", "1.0", "exmodz-file", []byte("bear-bytes")) + pm := svc.NewProfileManager() + require.NoError(t, pm.SetModLock(game.ID, "default", "fake-compiler", "bear-mount", "")) + + _, err := svc.ApplyMergedPakRegen(context.Background(), game, "default", nil) + require.NoError(t, err, "ApplyMergedPakRegen must never refuse due to a lock - #196's ErrModLocked gate does not apply here") +} +``` + +- [ ] **Step 2: Run the tests** + +Run: `go test ./internal/core/... -run 'TestLockedMod' -v` +Expected: all 4 PASS, WITHOUT any change to `merged_pak.go` — this task is pure verification that Task 6/9's implementation already has the intended (no lock-gate) behavior, since design decision 3 was already the plan going into Task 6/9's own writing. If any of these 4 fail, STOP and re-examine `syncMergedPak`/`ApplyMergedPakRegen`/`CheckMergedPakStaleness` for an accidental lock-gate check that shouldn't be there (there should be NONE — `grep -n "Locked\|ErrModLocked" internal/core/merged_pak.go` should return zero matches). + +- [ ] **Step 3: Commit** + +```bash +git add internal/core/merged_pak_locked_test.go +git commit -m "test: pin locked-mod-does-not-block-merge semantics (#197, coordinator-flagged design decision)" +``` + +### Task 14: CHANGELOG amendment + final full-repo verification + +**Files:** + +- Modify: `CHANGELOG.md` (amend the #196 `[Unreleased]` bullet in place) + +**Interfaces:** + +- Consumes: nothing. +- Produces: nothing — this is the plan's closing task. + +- [ ] **Step 1: Amend the CHANGELOG bullet** + +`CHANGELOG.md`'s `[Unreleased] / Added` section currently reads (in full, current text): + +``` +- Compiled mods (`deploy_mode: compile`, e.g. Icarus) now recover automatically when the game's base `data.pak` changes underneath them — "the Friday problem": a weekly base-pak refresh used to silently revert a compiled mod's patched tables, with nothing to notice. Compiling now records the base pak's footer fingerprint and retains a copy of the original `.exmodz` beside the compiled `_P.pak`, both invisible to deployment. `lmm update` (CLI and TUI) checks every compiled mod's fingerprint against the game's current base pak and reports a same-version "recompile needed" row (additive `--json` field `recompile_needed`/`reason`) alongside normal version updates; applying it recompiles in place from the retained `.exmodz` (falling back to a re-download when possible) and redeploys — pinned mods recompile normally, locked mods are refused with the same loud lock warning a real update gets. Pre-existing compiled installs without the new fingerprint are left alone rather than guessed at (indistinguishable from a plain prebuilt `.pak`); they pick up fingerprinting on their next real recompile. `lmm verify` gains a matching "RECOMPILE NEEDED" warning row (`stale_compile`) (#196) +``` + +Replace it (same list position, same `#196` reference retained alongside the new `#197` — this bullet now describes work spanning both issues, since #197 supersedes #196's per-mod behavior before either ever shipped) with: + +``` +- Compiled mods (`deploy_mode: compile`, e.g. Icarus) with more than one enabled `.exmodz` mod now compose correctly instead of silently shadowing each other: every enabled mod's table-row diffs are applied sequentially, in profile load order, into ONE merged `zzz_LMM_Merged_P.pak` per profile (named to mount last, so it always wins over a plain prebuilt `.pak`'s own table override) — two mods patching different fields of the same row, or entirely different rows of the same table, both survive; only a genuine same-field conflict is last-wins, and a bundled-asset path collision (which can't compose) is last-wins with a loud warning. This also fixes "the Friday problem" (a weekly base-pak refresh silently reverting a mod's patched tables, with nothing to notice): the merge regenerates whenever the enabled-mod set, load order, a mod's version, or the base pak itself changes. `lmm update` (CLI and TUI) reports a "recompile needed" row for the profile's merged pak (additive `--json` field `recompile_needed`/`reason`) alongside normal version updates; applying it regenerates and redeploys — pinned mods' diffs recompile normally, and a LOCKED mod's diff still participates in every merge (a lock pins that mod's own version, not the profile's merged pak). Installing/importing a `.exmodz` now only validates and retains it (a per-mod compiled pak is no longer generated or deployed); a plain prebuilt `.pak` mod, and every non-`deploy_mode: compile` game, is completely unaffected. `lmm verify` gains a matching "RECOMPILE NEEDED" warning row (`stale_compile`) for the profile's merged pak (#136, #175, #196, #197) +``` + +- [ ] **Step 2: Run `make man` if any CLI `--help`/`Long` text changed** + +Task 10/11 did not change any cobra `Long`/`Short` help text (only internal call sites and rendering logic that was already generic) — confirm with `git diff --stat` against `cmd/lmm/update.go`/`cmd/lmm/verify.go`'s `Long:` string literals specifically; if genuinely unchanged, `make man` is a no-op and `go test ./cmd/lmm/... -run TestGenManTree_MatchesCommittedPages` stays green without regenerating. If ANY `Long`/`Short` text drifted during Task 10/11 (e.g. if an implementer added a `#197`-specific clarifying line while there), run `make man` and commit the regenerated `docs/man/` pages alongside this task's CHANGELOG commit. + +- [ ] **Step 3: Full-repo verification** + +Run, in order, stopping at the first failure: + +```bash +gofmt -l . && echo "gofmt clean" +go vet ./... && echo "vet clean" +go build ./... && echo "build clean" +go test ./... 2>&1 | tail -60 +trunk check --no-fix $(git diff --name-only c0ca7af..HEAD -- '*.go' | tr '\n' ' ') 2>&1 | tail -80 +``` + +(`c0ca7af` was `develop`'s tip immediately before #196's own branch point — replace with whatever this plan's actual base commit is if it differs by the time implementation starts; the intent is "every `.go` file this whole plan touched, from Task 1 through Task 13".) + +Expected: `gofmt`/`go vet`/`go build` all clean; `go test ./...` fully green across every package (`internal/source/icarus`, `internal/storage/cache`, `internal/core`, `cmd/lmm`, `internal/tui`, plus every OTHER untouched package unaffected); `trunk check` reports zero NEW issues (pre-existing issue counts from before this plan started are fine, per this repo's own established convention throughout #172/#173/#189/#190/#196's own review cycles). + +- [ ] **Step 4: Manual smoke check (documented, not automatable in this plan)** + +This plan cannot execute a real Icarus install (no live game install in CI/this environment) — note explicitly in the implementation report whether a manual smoke test against a real Icarus install was performed (install 2+ real `.exmodz` mods that patch the SAME table, confirm the deployed `zzz_LMM_Merged_P.pak` actually contains both mods' changes in-game) or whether this plan's automated test suite (Task 1's `MergeCompile` tests, Task 6's `syncMergedPak` tests, Task 7/8's hook tests) is the sole verification. Matches this repo's own established precedent (#136/#190's smoke-test call-outs) of being explicit about what WAS and WASN'T verified against the real game, never silently claiming parity with reality that wasn't checked. + +- [ ] **Step 5: Commit** + +```bash +git add CHANGELOG.md +git commit -m "docs: amend #196 CHANGELOG entry for merged-pak compilation (#197)" +``` + +--- + +## Self-Review + +**1. Spec coverage** (against the task's "APPROVED DESIGN" paragraph, point by point): + +- "merged-ONLY model... ONE merged pak per profile, generated at deploy time by applying all enabled mods' retained .EXMOD diffs sequentially" → Task 1 (merge engine), Task 6 (deploy-time generation), Task 2/3 (per-mod paks no longer generated). ✅ +- "profile load order = upsert order; sequential upserts give field-level merge" → Task 1's `enabledExmodzSources`-fed, profile-load-order-preserving `[]source.MergeSource`; Task 1's `TestMergeCompile_FieldLevelMergeAcrossMods`/`TestMergeCompile_SameRowSameField_LastWins` extraction-verified. ✅ +- "same-path ASSET collisions = last-wins with a loud warning" → Task 1's asset-collision branch + `TestMergeCompile_AssetCollision_LastWinsWithWarning`, propagated through Task 6/9's `warnings` plumbing to CLI/TUI. ✅ +- "Per-mod \_P.pak artifacts are no longer generated or deployed" → Task 2/3. ✅ +- "install still parses/validates the .exmodz early + retains source + fingerprint in cache" → Task 2/3's `ValidateSource` + `cache.RetainedSourceName` retention; "fingerprint" clarified as the MERGED-level fingerprint (Task 5/6), not a per-mod one (Design Decision 5). ✅ +- "Merged pak name sorts LAST in mods/... pick exact name, justify" → Design Decision 1 + Task 5's `mergedPakFileName`. ✅ +- "Regeneration triggers: mod set/enable/disable/load order/mod version/base pak change" → Task 6's `syncMergedPak` fingerprint (all 5 dimensions, each with its own extraction-verified test in Task 5); Task 7/8's 8 hook call sites. ✅ +- "update shows re-merge rows" → Task 9/10/12. ✅ +- "verify checks the merged artifact" → Task 11. ✅ +- "locked mods: decide + justify semantics... propose, flag for coordinator" → Design Decision 3 + Task 13. ✅ +- "Provenance: the merged pak is a PROFILE-level artifact — design its deployed-file ownership/tracking so uninstall-to-zero removes it and #168-class stale-link hygiene is not worsened" → Design Decision 2 + Task 5 (synthetic mod identity) + Task 6's zero-sources uninstall branch + `TestSyncMergedPak_ZeroEnabledMods_UninstallsExistingPak`/`TestDisableMod_.../TestUninstallMod_...` (Task 7). ✅ +- "Plain-pak mods and non-compile games byte-unchanged" → Task 2/3's branches are additive (only the `isExmodzFile` branch changes; the `DeployCopy`/extract branches are untouched); Task 6/9's `game.DeployMode != domain.DeployCompile` guards; explicitly tested (`TestSyncMergedPak_NonCompileGame_NoOp`, `TestCheckMergedPakStaleness_NonCompileGame_Nil`). ✅ + +**2. Placeholder scan:** every code block in every task is complete, runnable Go (or a literal shell command) — no `TODO`/`...`/"add appropriate handling" appears in any Step's implementation code. Where a step says "read X first" or "confirm via grep", that grep/read is itself the concrete instruction (find the exact current line range before editing), not a stand-in for missing content. + +**3. Type consistency:** `source.MergeSource`/`icarus.MergeSource` (Task 1's alias fix, extraction-verified), `MergedFingerprint`/`MergedFingerprintEntry` (Task 5, used identically in Task 6/9/13), `Service.syncMergedPak`/`CheckMergedPakStaleness`/`ApplyMergedPakRegen`/`enabledExmodzSources`/`currentMergedFingerprint` (introduced Task 5/6/9, consumed identically throughout Task 7/8/9/10/11/12/13 with no signature drift), `mergedPakModID`/`mergedPakVersion`/`mergedPakFileName` (Task 5, referenced by exact name in Task 6/9/10/11/12/13's tests) — all consistent across every task that references them. diff --git a/docs/plans/archive/2026-08-01-icarus-quickbms-fallback-design.md b/docs/plans/archive/2026-08-01-icarus-quickbms-fallback-design.md new file mode 100644 index 0000000..a87c044 --- /dev/null +++ b/docs/plans/archive/2026-08-01-icarus-quickbms-fallback-design.md @@ -0,0 +1,62 @@ +# Icarus: QuickBMS Auto-Extraction Fallback — Design + +**Issue:** [#174](https://github.com/DonovanMods/linux-mod-manager/issues/174) · **Epic:** #136 (branch `epic/icarus-136`; PR #171 merged there) · **Status:** design approved by user 2026-08-01; implementation gated on the Task-1 spike. + +## Problem + +The compile pipeline's base-table chain (`data_dump_path` local dir → hosted community dump → loud failure) is at the mercy of the hosted dump repo's freshness (currently Week 236 vs the installed Week 243). Users with QuickBMS available should never hit that wall: the installed `data.pak` itself is always week-correct base truth — it just needs Oodle-capable extraction, which QuickBMS (the dump-repo maintainer's own tool) provides. + +## Decisions (user, 2026-08-01) + +1. **Auto-run + announce** when needed; `auto_extract: false` opts out. No interactive prompt. +2. **`.bms` script ships embedded** (go:embed), pending the spike's license check; if embedding is legally murky, fall back to download-on-demand from the canonical source with caching. +3. This is lmm's **first sanctioned external-binary invocation** — optional, runtime-detected, announced; never a hard dependency. +4. Lives on the **epic branch** (`epic/icarus-136`) as its own story PR; the epic merges to develop as one PR when complete. +5. QuickBMS is **not yet installed** on the reference machine — the spike performs a user-local (non-root) build first and records the recommended permanent install route. + +## Fallback chain + +Base-table acquisition (inside the icarus source's provider, before `Compile` writes anything): + +1. `data_dump_path` local dir — explicit user override, highest priority. +2. **Cached QuickBMS extraction for this exact build** (new) — `/icarus/extracted//`, populated by a previous auto-run; cheap disk check. +3. Hosted community dump (existing). +4. **QuickBMS auto-run** (new) — binary via `exec.LookPath("quickbms")` or `quickbms_path` config; announce the exact command and reason; extract the installed `data.pak` into the per-build cache dir; normalize layout; proceed. +5. All failed → one error enumerating every source tried, why each failed, and the remedies (set `data_dump_path`, install QuickBMS, wait for the dump repo). + +**Every source passes the same `validateDump` byte-compare gate** (40 stored tables vs the installed pak). QuickBMS output is week-correct by construction (it reads the installed pak); the gate proves the extraction wasn't mangled. Ordering rationale: hosted dump first keeps the zero-dependency path primary per the user's "if/when the files are out of date" framing; the per-build cache (step 2) makes extraction a once-per-game-update cost. + +## Component + +`internal/source/icarus/quickbms.go` — one file, four responsibilities: + +- **Detection:** `quickbms_path` config override, else `exec.LookPath`. Absence is a normal chain-miss, not an error. +- **Invocation:** embedded UE4 `.bms` script written to a temp file; QuickBMS executed with `exec.CommandContext` (timeout), stdout/stderr captured; non-zero exit → wrapped, actionable error carrying the tail of the tool's output. +- **Normalization:** map QuickBMS's output layout (pinned by the spike) into the dump-tree shape `loadLocalDump` already consumes. +- **Cache:** per-build directory under the data dir (`SetDataDir`'s directory — previously reserved, now used); stale builds' caches are ignored (validation would reject them anyway) and may be pruned opportunistically. + +`Compile`'s exported signature is unchanged; the chain slots into the existing provider logic. + +## Config (games.yaml, beside `data_dump_path`) + +- `auto_extract: true` (default) — set `false` to disable the auto-run. +- `quickbms_path: /path/to/quickbms` (optional) — for non-PATH installs (including the spike's user-local build). + +No CLI/TUI surface: pipeline-internal, announced through existing progress/logging. (CLI/TUI parity holds trivially — shared core path.) + +## Error handling + +Fail-loud throughout (repo precedent #95): extraction errors, validation failures, and missing tools each produce specific, remedial messages; the chain-exhausted error names all attempts. Partial extraction output is removed on failure (same hygiene as `Compile`'s partial-pak cleanup). Never silently fall back _across weeks_ — a stale-but-validating source is impossible by construction of the gate. + +## Testing + +- **Hermetic CI:** a stub `quickbms` executable on `$PATH` (test-written script emitting a known tree) covers detection, invocation, normalization, validation pass/fail, cache reuse, `auto_extract: false`, missing-binary, non-zero-exit, and timeout legs. No network, no real QuickBMS, no real game files. +- **Real validation:** spike + post-plan manual steps on the reference machine (extract real `data.pak`, byte-compare, then an end-to-end `Compile` of the real `Bear_Mount.EXMODZ` — which this feature finally unblocks). + +## Task-1 spike (the gate) + +On the reference machine: user-local QuickBMS build (no root); obtain the ecosystem UE4 `.bms` script; extract the real `data.pak`; verify Oodle tables decompress (byte-compare the 40 stored tables against `unrealpak` reads, spot-check JSON validity of previously unreachable tables like `D_ItemsStatic.json`); pin the exact invocation, output layout, and runtime; check the script's license for embedding. **If Linux QuickBMS cannot decompress Icarus's Oodle tables, stop — the design's premise is falsified and the feature is rethought before any product code.** Also record the recommended permanent install route for the user (AUR vs upstream). + +## Out of scope + +Extraction for any other game; any non-QuickBMS extractor; dump self-hosting; prompting UIs; making QuickBMS a required dependency. diff --git a/docs/plans/archive/2026-08-01-icarus-quickbms-fallback.md b/docs/plans/archive/2026-08-01-icarus-quickbms-fallback.md new file mode 100644 index 0000000..10cbca6 --- /dev/null +++ b/docs/plans/archive/2026-08-01-icarus-quickbms-fallback.md @@ -0,0 +1,2024 @@ +# QuickBMS Auto-Extraction Fallback Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** When the hosted community base-table dump is stale (or absent), compile Icarus mods anyway by extracting the installed `data.pak` with QuickBMS — the one always-week-correct source of base truth — automatically, announced, and behind the same validation gate as every other source. + +**Architecture:** A new `internal/source/icarus/quickbms.go` owns detection (`quickbms_path` config → `exec.LookPath`), invocation (`exec.CommandContext` with a timeout, driving an embedded UE4 `.bms` script), normalization of QuickBMS's output into the dump-tree shape `loadLocalDump` already consumes, and a per-build extraction cache under the service data dir. `DumpStore.DumpForBuild` becomes an explicit four-step chain — `data_dump_path` → per-build extraction cache → hosted dump → QuickBMS auto-run — with every step passing the existing `validateDump` byte-compare gate and a single exhaustive error when all fail. `Compile`'s call flow is untouched; the two new per-game settings reach it through a `source.CompileRequest` struct that replaces the `Compiler` interface's positional path arguments. + +**Tech Stack:** Go 1.25.6 (this repo's version), stdlib only for lmm code (`os/exec`, `embed`, `context`, `archive/tar` already in use) — no new third-party dependencies. QuickBMS itself is an external, optional, runtime-detected binary; its `.bms` script ships via `go:embed` (pending Task 1's license check). + +## Global Constraints + +Binding rules, copied from the approved design ([`2026-08-01-icarus-quickbms-fallback-design.md`](2026-08-01-icarus-quickbms-fallback-design.md)): + +- **Stdlib only for lmm code.** `go.mod` gains nothing. `os/exec` and `embed` are stdlib; QuickBMS is an external binary, not a Go dependency. +- **Fail loud, no silent fallbacks** (repo precedent #95). Extraction errors, validation failures, and missing tools each produce specific, remedial messages. The chain-exhausted error names every attempt and why it failed. Partial extraction output is removed on failure — the same hygiene as `Compile`'s partial-pak cleanup. +- **The external binary is optional, runtime-detected, and announced — never required.** Absence of QuickBMS is a normal chain-miss, not an error. This is lmm's first sanctioned external-binary invocation; it must not become a hard dependency of building, testing, or running lmm. +- **`auto_extract` defaults to `true`.** Setting it `false` opts out of the auto-run leg entirely. There is no interactive prompt. +- **Every source passes the same `validateDump` gate** (byte-compare of the 40 stored tables against the installed pak). QuickBMS output is week-correct by construction; the gate proves the extraction wasn't mangled. Never silently fall back _across weeks_ — a stale-but-validating source is impossible by construction of the gate. +- **No CLI or TUI surface.** Both new settings are `games.yaml`-only; the feature is pipeline-internal and announced through the existing logging/writer pattern. CLI/TUI parity holds trivially — shared core path, no new capability to surface in either. +- **Epic-branch workflow.** This is a story on `epic/icarus-136`: branch from it, PR back into it with `--base epic/icarus-136`, and reference [#174](https://github.com/DonovanMods/linux-mod-manager/issues/174) in commits and the PR. The epic merges to `develop` as one PR when complete. No version bump in this story. +- **Out of scope** (from the design, restated so it is not rediscovered mid-implementation): extraction for any other game; any non-QuickBMS extractor; dump self-hosting; prompting UIs; making QuickBMS a required dependency. + +### `SPIKE-CONFIRM:` markers + +Task 1 pins facts that Tasks 2–7 encode as best assumptions. Every such site carries a single-line `SPIKE-CONFIRM:` comment. After Task 1, `grep -rn 'SPIKE-CONFIRM' docs/plans/2026-08-01-icarus-quickbms-fallback.md internal/` finds all of them for the revision pass. Do not invent a second tag spelling. + +--- + +## Task 1: Empirical spike — build QuickBMS, extract the real `data.pak`, verify Oodle + +**This is the gate. No product code in this task.** If Linux QuickBMS cannot decompress Icarus's Oodle-compressed tables, **STOP**: the design's premise is falsified and the feature must be rethought before any of Tasks 2–7 begin. + +**Files:** + +- Create: `docs/plans/icarus-quickbms-spike-findings.md` (scratch findings doc, gitignored alongside the other `docs/plans/*` in-flight docs — do not `git add`) + +**Interfaces:** + +- Produces (consumed by Tasks 2–4 as the `SPIKE-CONFIRM:` answers): the exact QuickBMS invocation and flag order, the `.bms` script's canonical URL + filename + license verdict, QuickBMS's output directory layout, extraction wall-clock runtime, and the recommended permanent install route. + +- [ ] **Step 1: Check for a packaged QuickBMS (do NOT install, do NOT sudo)** + +Record availability only — the spike builds user-locally regardless, so nothing here needs root. + +```bash +# Arch/CachyOS: is it in the AUR? +curl -s 'https://aur.archlinux.org/rpc/v5/search/quickbms' | python3 -m json.tool | head -40 +# Is anything already on PATH? +command -v quickbms || echo "quickbms not on PATH (expected: not yet installed)" +``` + +Record: AUR package name(s) if any, their version and last-updated date, and whether a binary was already present. + +- [ ] **Step 2: Build QuickBMS user-locally (no root)** + +QuickBMS is Luigi Auriemma's tool; the Linux build is a plain `make`. Build under the scratch dir, never into `/usr`. + +```bash +WORK="$HOME/.local/src/quickbms" +mkdir -p "$WORK" && cd "$WORK" +curl -sL -o quickbms.zip https://aluigi.altervista.org/papers/quickbms.zip +unzip -o -q quickbms.zip +# Build dependencies are vendored in the tarball; the Makefile targets Linux directly. +make 2>&1 | tail -20 +ls -la quickbms +./quickbms 2>&1 | head -5 # prints version banner + usage +``` + +Record: the exact commands that worked, any build errors and their fixes, the resulting binary path, and the version banner. If `make` fails for missing system headers, record exactly which — a build that needs root-installed `-dev` packages is a material finding for the "optional dependency" story. + +Then install it where the config will point at it, still without root: + +```bash +mkdir -p "$HOME/.local/bin" && cp "$WORK/quickbms" "$HOME/.local/bin/quickbms" +command -v quickbms # ~/.local/bin is already on this machine's PATH per the dotfiles +``` + +- [ ] **Step 3: Obtain the ecosystem UE4 `.bms` script and check its license** + +The dump repo `GODOFMINECRAFT4/IcarusData` — the source Part 3 of the findings doc identified — commits its extraction toolchain alongside the tables, including the `.bms` script it drives. That is the ecosystem-proven script for this exact pak. + +```bash +cd "$WORK" +curl -sL -o unreal_pak.bms \ + "https://raw.githubusercontent.com/GODOFMINECRAFT4/IcarusData/master/unreal_pak.bms" +wc -l unreal_pak.bms && head -40 unreal_pak.bms +# The canonical upstream (aluigi's script index) for provenance + license comparison: +curl -sL -o unreal_pak.upstream.bms "https://aluigi.altervista.org/bms/unreal_tournament_4.bms" +diff -u unreal_pak.upstream.bms unreal_pak.bms | head -40 || true +``` + +**License check (this decides the embedding strategy):** read the script's header comment block and the upstream page for a license statement. Record the verdict explicitly as one of: + +- **EMBED OK** — the script carries a permissive/public-domain grant, or none plus an explicit "free to use" statement. Tasks 2–3 keep `go:embed`. +- **MURKY** — no grant, or terms that restrict redistribution. Then Tasks 2–3 switch to the design's fallback: download-on-demand from the canonical URL into the cache dir, with the URL and checksum pinned in code. Note this flips `SPIKE-CONFIRM:` sites in Task 2 Step 3 and Task 3 Step 3. + +Record the exact license text found (or "none present"), the URL it came from, and the verdict. + +- [ ] **Step 4: Extract the real `data.pak`** + +```bash +PAK=/data/SteamLibrary/steamapps/common/Icarus/Icarus/Content/Data/data.pak +OUT="$WORK/extracted" +rm -rf "$OUT" && mkdir -p "$OUT" +time quickbms "$WORK/unreal_pak.bms" "$PAK" "$OUT" 2>&1 | tail -30 +``` + +Record verbatim: the full command, its exit code, the tail of its output, and the wall-clock time. + +Then map the output layout precisely — Task 3's normalization depends on it: + +```bash +find "$OUT" -name '*.json' | wc -l # expect 298 +find "$OUT" -name 'DataTableMetadata.json' # the pak's root-level table: reveals the tree root +find "$OUT" -maxdepth 3 -type d | head -20 # is the tree nested under a mount-path prefix? +find "$OUT" -name 'D_Factions.json' +find "$OUT" -name 'D_ItemsStatic.json' -exec ls -la {} \; +``` + +Record: the number of `.json` files, the absolute path of `DataTableMetadata.json`, and whether the table tree sits at `$OUT` directly or beneath a prefix (the pak's mount point is the absolute cook path `C:/BA/work/92bbbfa44df12262/Temp/Data/`, so QuickBMS may recreate part of it). + +- [ ] **Step 5: Verify against `unrealpak` ground truth** + +Two assertions decide the gate. Run from the repo so `internal/unrealpak` is importable. + +```bash +cd /home/dyoung/Projects/orca/workspaces/linux-mod-manager/icarus-136 +mkdir -p /tmp/qbms-spike && cat > /tmp/qbms-spike/main.go <<'EOF' +package main + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" +) + +func main() { + pakPath, outDir := os.Args[1], os.Args[2] + pak, err := unrealpak.Open(pakPath) + if err != nil { + panic(err) + } + defer pak.Close() + + var stored, storedMatch, oodle, oodleFound, oodleValidJSON int + var mismatches []string + for _, f := range pak.Files() { + extracted := filepath.Join(outDir, filepath.FromSlash(f.Path)) + got, readErr := os.ReadFile(extracted) + shipped, pakErr := pak.ReadFile(f.Path) + switch { + case errors.Is(pakErr, unrealpak.ErrUnsupportedFormat): // Oodle: previously unreachable + oodle++ + if readErr != nil { + mismatches = append(mismatches, "OODLE NOT EXTRACTED: "+f.Path) + continue + } + oodleFound++ + var v any + if json.Unmarshal(bytes.TrimPrefix(got, []byte("\xef\xbb\xbf")), &v) == nil { + oodleValidJSON++ + } else { + mismatches = append(mismatches, "INVALID JSON: "+f.Path) + } + continue + case pakErr != nil: + mismatches = append(mismatches, "PAK READ ERROR: "+f.Path+": "+pakErr.Error()) + continue + } + stored++ + if readErr != nil { + mismatches = append(mismatches, "MISSING: "+f.Path) + continue + } + // Extraction may write LF or CRLF; the pak stores CRLF. Compare in the + // pak's own shape, exactly as toCRLF does at ingest. + norm := strings.ReplaceAll(strings.ReplaceAll(string(got), "\r\n", "\n"), "\n", "\r\n") + if !bytes.Equal([]byte(norm), shipped) { + mismatches = append(mismatches, "DIFFERS: "+f.Path) + continue + } + storedMatch++ + } + fmt.Printf("stored tables: %d byte-identical after CRLF normalization: %d\n", stored, storedMatch) + fmt.Printf("oodle tables: %d extracted: %d valid JSON: %d\n", oodle, oodleFound, oodleValidJSON) + for _, m := range mismatches { + fmt.Println(" " + m) + } + if storedMatch == stored && oodleFound == oodle && oodleValidJSON == oodle { + fmt.Println("GATE: PASS") + return + } + fmt.Println("GATE: FAIL") +} +EOF +go run /tmp/qbms-spike/main.go \ + /data/SteamLibrary/steamapps/common/Icarus/Icarus/Content/Data/data.pak \ + "$HOME/.local/src/quickbms/extracted" +``` + +**Acceptance:** + +- All **40** stored tables byte-identical to `unrealpak` reads (after CRLF normalization). +- All **258** Oodle tables extracted and parsing as valid JSON — including `Items/D_ItemsStatic.json` (~7.3 MB), which `unrealpak` cannot read at all. + +Spot-check the headline table by hand too: + +```bash +ls -la "$HOME/.local/src/quickbms/extracted"/**/D_ItemsStatic.json 2>/dev/null || \ + find "$HOME/.local/src/quickbms/extracted" -name D_ItemsStatic.json -exec ls -la {} \; +find "$HOME/.local/src/quickbms/extracted" -name D_ItemsStatic.json -exec sh -c \ + 'python3 -m json.tool < "$1" | head -5' _ {} \; +``` + +- [ ] **Step 6: STOP-AND-REVISE GATE** + +If `GATE: FAIL` — specifically if the Oodle tables did **not** decompress — **stop here**. Do not start Task 2. Write the findings doc with exactly what was observed (QuickBMS version, script, command, error output) and report the premise as falsified: the design's core claim is that Linux QuickBMS provides Oodle decompression, and without it this feature cannot exist in this shape. + +If stored tables mismatch but Oodle works, that is a normalization problem, not a falsified premise: record the exact difference (line endings? BOM? trailing newline?) and carry it into Task 3's normalization as a `SPIKE-CONFIRM:`-resolved detail. + +- [ ] **Step 7: Write the findings doc** + +Create `docs/plans/icarus-quickbms-spike-findings.md` recording, in this order: + +1. AUR/package availability (Step 1) and the **recommended permanent install route** for the user (AUR package vs. the user-local build), with the reasoning. +2. The exact build commands that worked, the binary path, and the version banner. +3. The `.bms` script: canonical URL, filename, size, and the **license verdict** (EMBED OK / MURKY) with the license text quoted. +4. The exact extraction invocation, exit code, output tail, and wall-clock runtime. +5. The output layout: where `DataTableMetadata.json` landed, whether a prefix directory wraps the tree, and the `.json` count. +6. The gate results: stored byte-identical count, Oodle extracted/valid-JSON count, and any mismatches. +7. A short "answers to `SPIKE-CONFIRM:`" section listing each marker from Tasks 2–7 and its resolved value, so the revision pass is mechanical. + +- [ ] **Step 8: Commit** + +The findings doc is gitignored (`docs/plans/*`), so there is nothing to commit for this task. Record completion in the task tracker instead and proceed to Task 2 only if the gate passed. + +--- + +## Task 2: `quickbms.go` — binary detection and the embedded script + +**Files:** + +- Create: `internal/source/icarus/quickbms.go` +- Create: `internal/source/icarus/quickbms_test.go` +- Create: `internal/source/icarus/embedded/unreal_pak.bms` (the Task 1 script; `SPIKE-CONFIRM:` filename and whether embedding is permitted at all) + +**Interfaces:** + +- Consumes: nothing from this package yet — detection is self-contained. +- Produces: `func findQuickBMS(configuredPath string) (string, error)`, `var errQuickBMSNotFound = errors.New(...)`, `func writeScriptTo(dir string) (string, error)`, `const quickbmsBinaryName`. Task 3 depends on all four. + +- [ ] **Step 1: Write the failing tests** + +```go +package icarus + +import ( + "errors" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// writeStubQuickBMS writes an executable stub named quickbms into dir and +// returns its path. body is the shell script body appended after the shebang; +// the caller controls exactly what the stub does with its arguments. +// +// The stub is /bin/sh, so these tests are Linux/Unix-only — which matches this +// project's target. Windows skips (see skipIfNoShell). +func writeStubQuickBMS(t *testing.T, dir, body string) string { + t.Helper() + p := filepath.Join(dir, "quickbms") + script := "#!/bin/sh\n" + body + if err := os.WriteFile(p, []byte(script), 0o755); err != nil { + t.Fatalf("writing stub quickbms: %v", err) + } + return p +} + +func skipIfNoShell(t *testing.T) { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("stub quickbms is a /bin/sh script; this project targets Linux") + } +} + +// prependToPATH puts dir at the FRONT of PATH for the test's duration, so the +// stub shadows any real quickbms the developer happens to have installed. +// +// It must prepend rather than replace: the stub is a /bin/sh script that calls +// mkdir, so a PATH containing only the stub's own directory would leave the +// stub unable to find its own utilities — it would exit 0 having silently +// written nothing, and the test would fail with a confusing "tables disagree" +// instead of an obvious "stub broke". Tests that need NO quickbms on PATH set +// PATH to a bare empty dir instead, which is safe because nothing executes. +func prependToPATH(t *testing.T, dir string) { + t.Helper() + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} + +func TestFindQuickBMS_ConfiguredPathWins(t *testing.T) { + skipIfNoShell(t) + dir := t.TempDir() + stub := writeStubQuickBMS(t, dir, "exit 0\n") + + // An empty PATH proves the configured path is used directly, not looked up. + t.Setenv("PATH", t.TempDir()) + + got, err := findQuickBMS(stub) + if err != nil { + t.Fatalf("findQuickBMS(%q): %v", stub, err) + } + if got != stub { + t.Errorf("findQuickBMS = %q, want the configured path %q", got, stub) + } +} + +func TestFindQuickBMS_ConfiguredPathMissing_IsActionable(t *testing.T) { + missing := filepath.Join(t.TempDir(), "nope", "quickbms") + + _, err := findQuickBMS(missing) + if err == nil { + t.Fatal("expected an error for a configured quickbms_path that does not exist, got nil") + } + // A wrong quickbms_path is user misconfiguration, not a chain-miss: it must + // NOT report as errQuickBMSNotFound, or the chain would quietly skip it. + if errors.Is(err, errQuickBMSNotFound) { + t.Error("a configured-but-missing quickbms_path must be a hard error, not a chain-miss") + } + if !strings.Contains(err.Error(), "quickbms_path") { + t.Errorf("error %q should name the quickbms_path setting", err) + } +} + +func TestFindQuickBMS_FallsBackToPATH(t *testing.T) { + skipIfNoShell(t) + dir := t.TempDir() + stub := writeStubQuickBMS(t, dir, "exit 0\n") + prependToPATH(t, dir) + + got, err := findQuickBMS("") + if err != nil { + t.Fatalf("findQuickBMS(\"\"): %v", err) + } + if got != stub { + t.Errorf("findQuickBMS = %q, want the PATH-resolved stub %q", got, stub) + } +} + +func TestFindQuickBMS_AbsentFromPATH_IsChainMiss(t *testing.T) { + t.Setenv("PATH", t.TempDir()) // empty dir: nothing to find + + _, err := findQuickBMS("") + if !errors.Is(err, errQuickBMSNotFound) { + t.Fatalf("findQuickBMS error = %v, want errQuickBMSNotFound (a normal chain-miss)", err) + } +} + +func TestWriteScriptTo_ProducesAUsableScriptFile(t *testing.T) { + dir := t.TempDir() + + p, err := writeScriptTo(dir) + if err != nil { + t.Fatalf("writeScriptTo: %v", err) + } + body, err := os.ReadFile(p) + if err != nil { + t.Fatalf("reading written script: %v", err) + } + if len(body) == 0 { + t.Fatal("written .bms script is empty; the embedded script did not make it into the binary") + } + if filepath.Dir(p) != dir { + t.Errorf("script written to %q, want it inside %q", p, dir) + } +} +``` + +- [ ] **Step 2: Run to verify they fail (RED)** + +```bash +cd /home/dyoung/Projects/orca/workspaces/linux-mod-manager/icarus-136 +go test ./internal/source/icarus/... -run 'TestFindQuickBMS|TestWriteScriptTo' -v +``` + +Expected: FAIL — `findQuickBMS`, `errQuickBMSNotFound`, `writeScriptTo` undefined. + +- [ ] **Step 3: Implement detection + embedding in `quickbms.go`** + +First place the script from Task 1: + +```bash +mkdir -p internal/source/icarus/embedded +cp "$HOME/.local/src/quickbms/unreal_pak.bms" internal/source/icarus/embedded/unreal_pak.bms +``` + +```go +package icarus + +import ( + "embed" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" +) + +// quickbmsBinaryName is the executable looked up on PATH when quickbms_path is +// not configured. +// +// SPIKE-CONFIRM: Task 1 Step 2 pins the binary's installed name. QuickBMS also +// ships a "quickbms_4gb_files" variant for >4 GB inputs; Icarus's data.pak is +// 2.4 MB, so the base binary is the right default. +const quickbmsBinaryName = "quickbms" + +// embeddedScriptName is the .bms script driving the extraction. +// +// SPIKE-CONFIRM: Task 1 Step 3 pins the filename AND whether embedding is +// permitted. If the license verdict is MURKY, this constant and the go:embed +// directive below are replaced by download-on-demand into the cache dir, +// pinning the canonical URL and a content checksum. +const embeddedScriptName = "unreal_pak.bms" + +//go:embed embedded/unreal_pak.bms +var embeddedScripts embed.FS + +// errQuickBMSNotFound reports that no QuickBMS binary is available. This is a +// normal chain-miss — the tool is optional (see Global Constraints) — so the +// base-table chain records it as one attempt among several rather than +// aborting. A configured-but-wrong quickbms_path is deliberately NOT this +// error: that is user misconfiguration and must be loud. +var errQuickBMSNotFound = errors.New("quickbms not found") + +// findQuickBMS resolves the QuickBMS executable. A non-empty configuredPath +// (the game's quickbms_path) is used verbatim and must exist; otherwise PATH +// is searched. +func findQuickBMS(configuredPath string) (string, error) { + if configuredPath != "" { + info, err := os.Stat(configuredPath) + if err != nil { + return "", fmt.Errorf("icarus: the configured quickbms_path %s is not usable: %w", configuredPath, err) + } + if info.IsDir() { + return "", fmt.Errorf("icarus: the configured quickbms_path %s is a directory, not an executable", configuredPath) + } + return configuredPath, nil + } + found, err := exec.LookPath(quickbmsBinaryName) + if err != nil { + return "", fmt.Errorf("%w on PATH: %v", errQuickBMSNotFound, err) + } + return found, nil +} + +// writeScriptTo materializes the embedded .bms script inside dir and returns +// its path. QuickBMS takes a script file path, so the embedded bytes need a +// real file; callers pass a temp dir they own and clean up. +func writeScriptTo(dir string) (string, error) { + body, err := embeddedScripts.ReadFile("embedded/" + embeddedScriptName) + if err != nil { + return "", fmt.Errorf("icarus: reading embedded %s: %w", embeddedScriptName, err) + } + p := filepath.Join(dir, embeddedScriptName) + if err := os.WriteFile(p, body, 0o644); err != nil { + return "", fmt.Errorf("icarus: writing %s: %w", p, err) + } + return p, nil +} +``` + +- [ ] **Step 4: Run tests to verify they pass (GREEN)** + +```bash +go test ./internal/source/icarus/... -run 'TestFindQuickBMS|TestWriteScriptTo' -v +``` + +Expected: PASS for all five. + +- [ ] **Step 5: Commit** + +```bash +git add internal/source/icarus/quickbms.go internal/source/icarus/quickbms_test.go internal/source/icarus/embedded/ +git commit -m "feat: detect QuickBMS and embed the UE4 extraction script (#174)" +``` + +--- + +## Task 3: `quickbms.go` — invocation, normalization, and the per-build extraction cache + +**Files:** + +- Modify: `internal/source/icarus/quickbms.go` +- Modify: `internal/source/icarus/quickbms_test.go` + +**Interfaces:** + +- Consumes: `findQuickBMS`, `writeScriptTo` (Task 2); `loadLocalDump`, `Build` (existing `datadump.go`). +- Produces: `func extractedCacheDir(dataDir string, b Build) string`, `func runQuickBMS(ctx context.Context, exe, scriptPath, pakPath, rawDir string) error`, `func normalizeExtraction(rawDir, destDir string) error`, `func findTreeRoot(rawDir string) (string, error)`, `func extractWithQuickBMS(ctx context.Context, exe, pakPath, cacheDir string) (*Dump, error)`, `const quickbmsTimeout`, `const rootMarkerTable`. Task 4 depends on `extractedCacheDir` and `extractWithQuickBMS`; Tasks 4 and 6 use `rootMarkerTable` in their stub fixtures. + +- [ ] **Step 1: Write the failing tests** + +Append to `quickbms_test.go`: + +```go +// stubEmitting builds a stub-quickbms body that recreates files under the +// output directory ($3). Each entry is written with printf so the exact bytes +// (including CRLF) are reproducible from a POSIX shell. +func stubEmitting(files map[string]string) string { + body := "out=\"$3\"\n" + // Deterministic order keeps the generated script stable across runs. + names := make([]string, 0, len(files)) + for name := range files { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + esc := strings.ReplaceAll(files[name], "\\", "\\\\") + esc = strings.ReplaceAll(esc, "%", "%%") + esc = strings.ReplaceAll(esc, "\r", "\\r") + esc = strings.ReplaceAll(esc, "\n", "\\n") + esc = strings.ReplaceAll(esc, "\"", "\\\"") + body += fmt.Sprintf("mkdir -p \"$out/%s\"\n", filepath.Dir(name)) + body += fmt.Sprintf("printf \"%s\" > \"$out/%s\"\n", esc, name) + } + body += "exit 0\n" + return body +} + +func TestExtractedCacheDir_IsPerBuild(t *testing.T) { + b := Build{Major: 3, Minor: 0, Patch: 21, Changelist: 155335} + got := extractedCacheDir("/data", b) + want := filepath.Join("/data", "icarus", "extracted", "3.0.21.155335") + if got != want { + t.Errorf("extractedCacheDir = %q, want %q", got, want) + } +} + +func TestExtractWithQuickBMS_ProducesADumpAndPopulatesTheCache(t *testing.T) { + skipIfNoShell(t) + const rel = "Factions/D_Factions.json" + const body = "{\r\n \"Rows\": []\r\n}" + binDir := t.TempDir() + stub := writeStubQuickBMS(t, binDir, stubEmitting(map[string]string{ + rel: body, + "DataTableMetadata.json": "{}", + })) + cacheDir := filepath.Join(t.TempDir(), "extracted", "3.0.21.155335") + + dump, err := extractWithQuickBMS(context.Background(), stub, "/nonexistent/data.pak", cacheDir) + if err != nil { + t.Fatalf("extractWithQuickBMS: %v", err) + } + got, ok := dump.Table(rel) + if !ok { + t.Fatalf("dump has no table %q", rel) + } + if string(got) != body { + t.Errorf("table bytes = %q, want %q", got, body) + } + // The cache directory must be left populated for the next run to reuse. + if _, err := os.Stat(filepath.Join(cacheDir, filepath.FromSlash(rel))); err != nil { + t.Errorf("cache dir was not populated: %v", err) + } +} + +// QuickBMS may recreate part of the pak's mount path above the table tree. +// Normalization must find the real root, identified by the pak's known +// root-level table. +func TestExtractWithQuickBMS_StripsAMountPathPrefix(t *testing.T) { + skipIfNoShell(t) + const rel = "Factions/D_Factions.json" + binDir := t.TempDir() + const prefix = "C/BA/work/Temp/Data/" + stub := writeStubQuickBMS(t, binDir, stubEmitting(map[string]string{ + prefix + rel: "{\r\n}", + prefix + rootMarkerTable: "{}", + })) + cacheDir := filepath.Join(t.TempDir(), "extracted", "b") + + dump, err := extractWithQuickBMS(context.Background(), stub, "/nonexistent/data.pak", cacheDir) + if err != nil { + t.Fatalf("extractWithQuickBMS: %v", err) + } + if _, ok := dump.Table(rel); !ok { + t.Fatalf("table %q not found after prefix stripping; tables = %v", rel, dumpKeys(dump)) + } +} + +func TestExtractWithQuickBMS_NonZeroExit_IsActionable(t *testing.T) { + skipIfNoShell(t) + binDir := t.TempDir() + stub := writeStubQuickBMS(t, binDir, "echo 'oodle plugin missing' >&2\nexit 3\n") + cacheDir := filepath.Join(t.TempDir(), "extracted", "b") + + _, err := extractWithQuickBMS(context.Background(), stub, "/nonexistent/data.pak", cacheDir) + if err == nil { + t.Fatal("expected an error when quickbms exits non-zero, got nil") + } + if !strings.Contains(err.Error(), "oodle plugin missing") { + t.Errorf("error %q should carry the tool's own output", err) + } + // Partial output must not survive as a poisoned cache. + if _, statErr := os.Stat(cacheDir); statErr == nil { + t.Error("cache dir must not exist after a failed extraction") + } +} + +func TestExtractWithQuickBMS_EmptyOutput_IsActionable(t *testing.T) { + skipIfNoShell(t) + binDir := t.TempDir() + stub := writeStubQuickBMS(t, binDir, "exit 0\n") // succeeds, writes nothing + cacheDir := filepath.Join(t.TempDir(), "extracted", "b") + + _, err := extractWithQuickBMS(context.Background(), stub, "/nonexistent/data.pak", cacheDir) + if err == nil { + t.Fatal("expected an error when quickbms produces no tables, got nil") + } + if _, statErr := os.Stat(cacheDir); statErr == nil { + t.Error("cache dir must not exist after an empty extraction") + } +} + +func TestExtractWithQuickBMS_Timeout(t *testing.T) { + skipIfNoShell(t) + binDir := t.TempDir() + stub := writeStubQuickBMS(t, binDir, "sleep 5\n") + cacheDir := filepath.Join(t.TempDir(), "extracted", "b") + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := extractWithQuickBMS(ctx, stub, "/nonexistent/data.pak", cacheDir) + if err == nil { + t.Fatal("expected a timeout error, got nil") + } + if elapsed := time.Since(start); elapsed > 3*time.Second { + t.Errorf("extraction took %v; the context deadline did not kill the process", elapsed) + } + if _, statErr := os.Stat(cacheDir); statErr == nil { + t.Error("cache dir must not exist after a timed-out extraction") + } +} + +// dumpKeys lists a dump's table paths, for failure messages. +func dumpKeys(d *Dump) []string { + out := make([]string, 0, len(d.tables)) + for k := range d.tables { + out = append(out, k) + } + sort.Strings(out) + return out +} +``` + +Add to `quickbms_test.go`'s imports: `"context"`, `"fmt"`, `"sort"`, `"time"`. + +- [ ] **Step 2: Run to verify they fail (RED)** + +```bash +go test ./internal/source/icarus/... -run 'TestExtract|TestExtractedCacheDir' -v +``` + +Expected: FAIL — `extractedCacheDir`, `extractWithQuickBMS`, `rootMarkerTable` undefined. + +- [ ] **Step 3: Implement invocation, normalization, and caching** + +Append to `quickbms.go` (and extend its import block with `"context"`, `"io/fs"`, `"strings"`, `"time"`): + +```go +// quickbmsTimeout bounds a single extraction. The real data.pak is 2.4 MB and +// extracts in seconds; this ceiling exists to kill a wedged process, not to +// pace a slow one. +// +// SPIKE-CONFIRM: Task 1 Step 4 records the real wall-clock runtime. If it is +// anywhere near this bound, raise it — a timeout that can fire on a healthy +// machine would turn a working feature into a flaky one. +const quickbmsTimeout = 10 * time.Minute + +// quickbmsWaitDelay bounds how long Wait blocks for output pipes *after* the +// context has already killed the process. Without it, a killed child that left +// a grandchild holding the inherited stdout/stderr pipe would hang +// CombinedOutput until that grandchild exits — the timeout would fire, the +// process would die, and the call would still block. Two seconds is generous +// for draining a pipe and only ever applies on the already-failing path. +const quickbmsWaitDelay = 2 * time.Second + +// rootMarkerTable is a table the pak stores at its own root. QuickBMS may +// recreate part of the pak's mount point (an absolute cook path, +// "C:/BA/work/.../Temp/Data/") above the table tree, so the tree's real root is +// found by locating this file rather than assuming a fixed depth. +// +// SPIKE-CONFIRM: Task 1 Step 4 records where DataTableMetadata.json actually +// landed. If QuickBMS writes the tree directly at the output root, this search +// still resolves correctly (depth 0) — the marker approach is layout-agnostic +// by design and needs no change either way. +const rootMarkerTable = "DataTableMetadata.json" + +// maxExtractedTableSize caps a single extracted table, mirroring +// maxTarEntrySize's role for the hosted dump. The largest real table is 7.3 MB. +const maxExtractedTableSize = 64 << 20 + +// extractedCacheDir is where a build's extracted tables live: +// /icarus/extracted//. Keying by build means a game update +// naturally misses the cache and re-extracts, and stale builds' directories are +// simply never consulted. +func extractedCacheDir(dataDir string, b Build) string { + return filepath.Join(dataDir, "icarus", "extracted", b.String()) +} + +// runQuickBMS invokes the tool, capturing its output for error reporting. +// +// SPIKE-CONFIRM: Task 1 Step 4 pins the exact argument order and any required +// flags. The documented QuickBMS form is +// `quickbms `; if the spike needed +// extra flags (e.g. -o to overwrite without prompting), add them here. +func runQuickBMS(ctx context.Context, exe, scriptPath, pakPath, rawDir string) error { + cmd := exec.CommandContext(ctx, exe, scriptPath, pakPath, rawDir) + cmd.WaitDelay = quickbmsWaitDelay + out, err := cmd.CombinedOutput() + if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return fmt.Errorf("icarus: QuickBMS extraction did not finish within %s: %w", quickbmsTimeout, ctxErr) + } + return fmt.Errorf("icarus: QuickBMS failed extracting %s: %w\n%s", pakPath, err, tailOutput(out)) + } + return nil +} + +// tailOutput trims a tool's captured output to its last few lines, which is +// where QuickBMS reports the actual failure. +func tailOutput(out []byte) string { + const maxLines = 15 + lines := strings.Split(strings.TrimRight(string(out), "\n"), "\n") + if len(lines) > maxLines { + lines = lines[len(lines)-maxLines:] + } + return strings.Join(lines, "\n") +} + +// normalizeExtraction copies QuickBMS's raw output into destDir in the +// dump-tree shape loadLocalDump consumes: table paths relative to destDir, +// with any mount-path prefix stripped. +func normalizeExtraction(rawDir, destDir string) error { + treeRoot, err := findTreeRoot(rawDir) + if err != nil { + return err + } + copied := 0 + err = filepath.WalkDir(treeRoot, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(d.Name(), ".json") { + return nil + } + info, err := d.Info() + if err != nil { + return err + } + if info.Size() > maxExtractedTableSize { + return fmt.Errorf("extracted table %s is %d bytes, exceeding the %d-byte cap", + p, info.Size(), maxExtractedTableSize) + } + rel, err := filepath.Rel(treeRoot, p) + if err != nil { + return err + } + body, err := os.ReadFile(p) + if err != nil { + return err + } + dst := filepath.Join(destDir, rel) + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + return err + } + if err := os.WriteFile(dst, body, 0o644); err != nil { + return err + } + copied++ + return nil + }) + if err != nil { + return fmt.Errorf("icarus: normalizing QuickBMS output: %w", err) + } + if copied == 0 { + return fmt.Errorf("icarus: QuickBMS produced no JSON tables under %s", rawDir) + } + return nil +} + +// findTreeRoot locates the directory holding rootMarkerTable — the real root of +// the extracted table tree, whatever prefix directories sit above it. The +// shallowest match wins, so a table tree that legitimately nests a same-named +// file deeper cannot displace the true root. +func findTreeRoot(rawDir string) (string, error) { + best := "" + bestDepth := -1 + err := filepath.WalkDir(rawDir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || d.Name() != rootMarkerTable { + return nil + } + dir := filepath.Dir(p) + depth := strings.Count(filepath.ToSlash(dir), "/") + if bestDepth == -1 || depth < bestDepth { + best, bestDepth = dir, depth + } + return nil + }) + if err != nil { + return "", fmt.Errorf("icarus: scanning QuickBMS output %s: %w", rawDir, err) + } + if best == "" { + return "", fmt.Errorf("icarus: QuickBMS output under %s has no %s — "+ + "the extraction did not produce a recognizable data.pak table tree", rawDir, rootMarkerTable) + } + return best, nil +} + +// extractWithQuickBMS runs a full extraction into cacheDir and returns the +// resulting tables. +// +// The extraction lands in a sibling staging directory and is renamed into +// cacheDir only on success, so cacheDir is either absent or complete — a +// half-written cache can never be mistaken for a usable one on the next run. +// Every failure path removes its partial output, the same hygiene Compile +// applies to a partial pak. +func extractWithQuickBMS(ctx context.Context, exe, pakPath, cacheDir string) (dump *Dump, err error) { + parent := filepath.Dir(cacheDir) + if err := os.MkdirAll(parent, 0o755); err != nil { + return nil, fmt.Errorf("icarus: preparing extraction cache %s: %w", parent, err) + } + staging, err := os.MkdirTemp(parent, ".extract-*") + if err != nil { + return nil, fmt.Errorf("icarus: preparing extraction staging: %w", err) + } + defer func() { + if err != nil { + _ = os.RemoveAll(staging) //nolint:errcheck // best-effort cleanup of a failed run + _ = os.RemoveAll(cacheDir) //nolint:errcheck // never leave a partial cache behind + } + }() + + rawDir := filepath.Join(staging, "raw") + if err = os.MkdirAll(rawDir, 0o755); err != nil { + return nil, fmt.Errorf("icarus: preparing extraction output dir: %w", err) + } + scriptPath, err := writeScriptTo(staging) + if err != nil { + return nil, err + } + + runCtx, cancel := context.WithTimeout(ctx, quickbmsTimeout) + defer cancel() + if err = runQuickBMS(runCtx, exe, scriptPath, pakPath, rawDir); err != nil { + return nil, err + } + + normalized := filepath.Join(staging, "tree") + if err = normalizeExtraction(rawDir, normalized); err != nil { + return nil, err + } + + // Publish atomically: the cache is complete the instant it exists. + if err = os.RemoveAll(cacheDir); err != nil { + return nil, fmt.Errorf("icarus: clearing stale extraction cache %s: %w", cacheDir, err) + } + if err = os.Rename(normalized, cacheDir); err != nil { + return nil, fmt.Errorf("icarus: publishing extraction cache %s: %w", cacheDir, err) + } + _ = os.RemoveAll(staging) //nolint:errcheck // best-effort: the useful output already moved + + dump, err = loadLocalDump(cacheDir) + if err != nil { + return nil, err + } + return dump, nil +} +``` + +- [ ] **Step 4: Run tests to verify they pass (GREEN)** + +```bash +go test ./internal/source/icarus/... -v +``` + +Expected: PASS, including every pre-existing test in the package. + +- [ ] **Step 5: Commit** + +```bash +git add internal/source/icarus/quickbms.go internal/source/icarus/quickbms_test.go +git commit -m "feat: run QuickBMS and cache its normalized output per build (#174)" +``` + +--- + +## Task 4: Chain integration — four ordered sources, one exhaustive error + +**Files:** + +- Modify: `internal/source/icarus/datadump.go` +- Modify: `internal/source/icarus/datadump_test.go` +- Modify: `internal/source/icarus/icarus.go` (`SetDataDir` now stores the directory) + +**Interfaces:** + +- Consumes: `extractedCacheDir`, `extractWithQuickBMS`, `findQuickBMS`, `errQuickBMSNotFound` (Tasks 2–3); `loadLocalDump`, `validateDump`, `detectBuild` (existing). +- Produces: `type chainInput struct{...}`, `func (s *DumpStore) DumpForBuild(ctx context.Context, in chainInput) (*Dump, error)` (replaces the 3-arg form), `func newDumpStore(httpClient *http.Client, dataDir string) *DumpStore`, `func installRootFromPak(basePakPath string) string`. Task 5 depends on `chainInput` and the new `DumpForBuild`. + +- [ ] **Step 1: Write the failing tests** + +Append to `datadump_test.go`. Existing `DumpForBuild` tests must also be migrated to the struct form — that migration is part of Step 3. + +```go +// newTestStore builds a DumpStore whose hosted leg points at srv and whose +// data dir is a temp dir, the shape every chain test needs. +func newTestStore(t *testing.T, treeURL, dataDir string) *DumpStore { + t.Helper() + s := newDumpStore(http.DefaultClient, dataDir) + s.treeURL = treeURL + return s +} + +// installWithPak lays out a minimal game install (version.json + data.pak) and +// returns the install root and the pak path. +func installWithPak(t *testing.T, tables map[string][]byte) (root, pakPath string) { + t.Helper() + root = t.TempDir() + cfg := filepath.Join(root, "Icarus", "Config") + if err := os.MkdirAll(cfg, 0o755); err != nil { + t.Fatal(err) + } + const vjson = `{"Name":"Icarus","Version":{"Major":3,"Minor":0,"Patch":21,` + + `"Changelist":155335,"BuildType":"Shipping","FeatureLevel":"DangerousHorizons"},` + + `"Data":{"Changelist":155151}}` + if err := os.WriteFile(filepath.Join(cfg, "version.json"), []byte(vjson), 0o644); err != nil { + t.Fatal(err) + } + dataDir := filepath.Join(root, "Icarus", "Content", "Data") + if err := os.MkdirAll(dataDir, 0o755); err != nil { + t.Fatal(err) + } + pakPath = filepath.Join(dataDir, "data.pak") + w, err := unrealpak.Create(pakPath) + if err != nil { + t.Fatalf("creating test base pak: %v", err) + } + for rel, body := range tables { + if err := w.AddFile(rel, body); err != nil { + t.Fatalf("AddFile(%q): %v", rel, err) + } + } + if err := w.Close(); err != nil { + t.Fatalf("closing test base pak: %v", err) + } + return root, pakPath +} + +func TestInstallRootFromPak(t *testing.T) { + got := installRootFromPak("/games/Icarus/Icarus/Content/Data/data.pak") + if want := "/games/Icarus"; got != want { + t.Errorf("installRootFromPak = %q, want %q", got, want) + } +} + +// Leg 2: a populated per-build cache is used without touching the network or +// running the tool. +func TestDumpForBuild_UsesPerBuildExtractionCache(t *testing.T) { + const rel = "Factions/D_Factions.json" + body := []byte("{\r\n \"Rows\": []\r\n}") + _, pak := installWithPak(t, map[string][]byte{rel: body}) + + dataDir := t.TempDir() + cache := extractedCacheDir(dataDir, Build{Major: 3, Patch: 21, Changelist: 155335}) + if err := os.MkdirAll(filepath.Join(cache, "Factions"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(cache, filepath.FromSlash(rel)), body, 0o644); err != nil { + t.Fatal(err) + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Error("hosted dump was fetched even though the per-build cache was populated") + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + store := newTestStore(t, srv.URL, dataDir) + dump, err := store.DumpForBuild(context.Background(), chainInput{basePakPath: pak, autoExtract: true}) + if err != nil { + t.Fatalf("DumpForBuild: %v", err) + } + if got, ok := dump.Table(rel); !ok || !bytes.Equal(got, body) { + t.Errorf("table = %q (found=%v), want %q", got, ok, body) + } +} + +// Leg 4: hosted dump stale -> QuickBMS runs, and its output is cached. +func TestDumpForBuild_AutoRunsQuickBMSWhenHostedDumpIsStale(t *testing.T) { + skipIfNoShell(t) + const rel = "Factions/D_Factions.json" + body := []byte("{\r\n \"Rows\": []\r\n}") + _, pak := installWithPak(t, map[string][]byte{rel: body}) + + // Hosted dump serves a different week's content: it must fail validation. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(tarGz(t, "IcarusData-old", map[string]string{rel: "{\n \"Rows\": [1]\n}"})) + })) + defer srv.Close() + + binDir := t.TempDir() + writeStubQuickBMS(t, binDir, stubEmitting(map[string]string{ + rel: "{\r\n \"Rows\": []\r\n}", + rootMarkerTable: "{}", + })) + prependToPATH(t, binDir) + + dataDir := t.TempDir() + store := newTestStore(t, srv.URL, dataDir) + + dump, err := store.DumpForBuild(context.Background(), chainInput{basePakPath: pak, autoExtract: true}) + if err != nil { + t.Fatalf("DumpForBuild: %v", err) + } + if got, ok := dump.Table(rel); !ok || !bytes.Equal(got, body) { + t.Errorf("table = %q (found=%v), want the extracted %q", got, ok, body) + } + cache := extractedCacheDir(dataDir, Build{Major: 3, Patch: 21, Changelist: 155335}) + if _, statErr := os.Stat(filepath.Join(cache, filepath.FromSlash(rel))); statErr != nil { + t.Errorf("extraction was not cached for reuse: %v", statErr) + } +} + +// An extraction that does not reproduce the installed pak's own stored tables +// is a mangled extraction, not a wrong week — it must fail loudly AND leave no +// cache behind for the next run to trust. +func TestDumpForBuild_ExtractionFailingValidation_IsRejectedAndNotCached(t *testing.T) { + skipIfNoShell(t) + const rel = "Factions/D_Factions.json" + _, pak := installWithPak(t, map[string][]byte{rel: []byte("{\r\n \"Rows\": []\r\n}")}) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(tarGz(t, "IcarusData-old", map[string]string{rel: "{\n \"Rows\": [1]\n}"})) + })) + defer srv.Close() + + // The stub emits a table whose bytes differ from the pak's own copy. + binDir := t.TempDir() + writeStubQuickBMS(t, binDir, stubEmitting(map[string]string{ + rel: "{\r\n \"Rows\": [99]\r\n}", + rootMarkerTable: "{}", + })) + prependToPATH(t, binDir) + + dataDir := t.TempDir() + store := newTestStore(t, srv.URL, dataDir) + + _, err := store.DumpForBuild(context.Background(), chainInput{basePakPath: pak, autoExtract: true}) + if err == nil { + t.Fatal("expected an error when the extraction does not match the installed pak") + } + cache := extractedCacheDir(dataDir, Build{Major: 3, Patch: 21, Changelist: 155335}) + if _, statErr := os.Stat(cache); statErr == nil { + t.Error("a cache that failed validation must be removed, not left for the next run to reuse") + } +} + +func TestDumpForBuild_AutoExtractFalse_SkipsQuickBMS(t *testing.T) { + skipIfNoShell(t) + const rel = "Factions/D_Factions.json" + _, pak := installWithPak(t, map[string][]byte{rel: []byte("{\r\n}")}) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(tarGz(t, "IcarusData-old", map[string]string{rel: "{\n \"x\": 1\n}"})) + })) + defer srv.Close() + + binDir := t.TempDir() + writeStubQuickBMS(t, binDir, "echo 'stub must not run' >&2\nexit 9\n") + prependToPATH(t, binDir) + + store := newTestStore(t, srv.URL, t.TempDir()) + _, err := store.DumpForBuild(context.Background(), chainInput{basePakPath: pak, autoExtract: false}) + if err == nil { + t.Fatal("expected the chain to fail with auto_extract disabled and a stale hosted dump") + } + if !strings.Contains(err.Error(), "auto_extract") { + t.Errorf("error %q should explain that auto_extract is disabled", err) + } +} + +func TestDumpForBuild_MissingBinary_ChainErrorNamesEveryAttempt(t *testing.T) { + const rel = "Factions/D_Factions.json" + _, pak := installWithPak(t, map[string][]byte{rel: []byte("{\r\n}")}) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(tarGz(t, "IcarusData-old", map[string]string{rel: "{\n \"x\": 1\n}"})) + })) + defer srv.Close() + t.Setenv("PATH", t.TempDir()) // no quickbms anywhere + + store := newTestStore(t, srv.URL, t.TempDir()) + _, err := store.DumpForBuild(context.Background(), chainInput{basePakPath: pak, autoExtract: true}) + if err == nil { + t.Fatal("expected an exhausted-chain error, got nil") + } + for _, want := range []string{"hosted", "QuickBMS", "data_dump_path"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("chain-exhausted error %q should mention %q", err, want) + } + } +} + +func TestDumpForBuild_ExplicitLocalDirFailure_DoesNotFallThrough(t *testing.T) { + const rel = "Factions/D_Factions.json" + _, pak := installWithPak(t, map[string][]byte{rel: []byte("{\r\n}")}) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Error("hosted dump was fetched despite an explicit data_dump_path") + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + local := t.TempDir() // exists, but holds no tables + store := newTestStore(t, srv.URL, t.TempDir()) + _, err := store.DumpForBuild(context.Background(), + chainInput{basePakPath: pak, localDumpDir: local, autoExtract: true}) + if err == nil { + t.Fatal("expected an error for an explicit but unusable data_dump_path") + } +} +``` + +Add to `datadump_test.go`'s imports: `"bytes"` (if absent). + +- [ ] **Step 2: Run to verify they fail (RED)** + +```bash +go test ./internal/source/icarus/... -run 'TestDumpForBuild|TestInstallRootFromPak' -v +``` + +Expected: FAIL — `chainInput`, `installRootFromPak`, and the 2-arg `newDumpStore` are undefined; existing `DumpForBuild` call sites no longer compile. + +- [ ] **Step 3: Implement the chain** + +In `datadump.go`, replace the `DumpStore` type, its constructor, and `DumpForBuild` with: + +```go +// DumpStore acquires base data tables for the installed game, trying every +// source in a fixed order (see DumpForBuild). It does not cache the hosted +// dump to disk; it does own the per-build QuickBMS extraction cache under +// dataDir. +type DumpStore struct { + httpClient *http.Client + dataDir string + treeURL string // overridable in tests +} + +func newDumpStore(httpClient *http.Client, dataDir string) *DumpStore { + return &DumpStore{httpClient: httpClient, dataDir: dataDir, treeURL: defaultDumpTreeURL} +} + +// chainInput carries everything the base-table chain needs for one compile. +// These are per-game settings, so they arrive per call rather than living on +// the store. +type chainInput struct { + basePakPath string // the installed game's data.pak + localDumpDir string // game.BaseDataPath (data_dump_path); "" when unset + autoExtract bool // game.AutoExtract (auto_extract); default true + quickbmsPath string // game.QuickBMSPath (quickbms_path); "" when unset +} + +// sourceAttempt records one leg of the chain for the exhausted-chain error. +type sourceAttempt struct { + name string + reason string +} + +// DumpForBuild returns base data tables that provably match the installed +// game, trying four sources in order: +// +// 1. data_dump_path — the explicit user override. If set, it is the ONLY +// source consulted: falling through from an override the user deliberately +// configured would hide their misconfiguration. +// 2. The per-build QuickBMS extraction cache — a previous auto-run's output +// for this exact build. A cheap disk check that makes extraction a +// once-per-game-update cost. +// 3. The hosted community dump — the zero-dependency path, kept ahead of the +// auto-run so users who need no external tool never invoke one. +// 4. A QuickBMS auto-run — extracts the installed pak, which is week-correct +// by construction. Skipped when auto_extract is false or no binary exists. +// +// Every source passes the same validateDump byte-compare gate; a source that +// fails it is recorded and the chain continues (legs 2-4), because those are +// derived or third-party sources rather than stated user intent. When all fail, +// one error enumerates every attempt and its reason. +func (s *DumpStore) DumpForBuild(ctx context.Context, in chainInput) (*Dump, error) { + // Leg 1: explicit override — no fallthrough. + if in.localDumpDir != "" { + dump, err := loadLocalDump(in.localDumpDir) + if err != nil { + return nil, err + } + if err := validateDump(dump, in.basePakPath); err != nil { + return nil, fmt.Errorf("%w (tables were read from the configured data_dump_path %s)", err, in.localDumpDir) + } + return dump, nil + } + + var attempts []sourceAttempt + + build, buildErr := detectBuild(installRootFromPak(in.basePakPath)) + cacheDir := "" + if buildErr == nil && s.dataDir != "" { + cacheDir = extractedCacheDir(s.dataDir, build) + } + + // Leg 2: per-build extraction cache. + switch { + case cacheDir == "": + attempts = append(attempts, sourceAttempt{"cached QuickBMS extraction", + fmt.Sprintf("no cache location available (%v)", buildErr)}) + default: + dump, err := loadLocalDump(cacheDir) + switch { + case err != nil: + attempts = append(attempts, sourceAttempt{"cached QuickBMS extraction", + fmt.Sprintf("no usable cache at %s", cacheDir)}) + default: + if vErr := validateDump(dump, in.basePakPath); vErr != nil { + attempts = append(attempts, sourceAttempt{"cached QuickBMS extraction", + fmt.Sprintf("cache at %s failed validation: %v", cacheDir, vErr)}) + } else { + return dump, nil + } + } + } + + // Leg 3: hosted community dump. + dump, err := s.fetchTree(ctx, s.treeURL) + switch { + case err != nil: + attempts = append(attempts, sourceAttempt{"hosted community dump", err.Error()}) + default: + if vErr := validateDump(dump, in.basePakPath); vErr != nil { + attempts = append(attempts, sourceAttempt{"hosted community dump", vErr.Error()}) + } else { + return dump, nil + } + } + + // Leg 4: QuickBMS auto-run. + switch { + case !in.autoExtract: + attempts = append(attempts, sourceAttempt{"QuickBMS auto-extraction", + "disabled by auto_extract: false in games.yaml"}) + case cacheDir == "": + attempts = append(attempts, sourceAttempt{"QuickBMS auto-extraction", + fmt.Sprintf("cannot determine the installed build to cache under (%v)", buildErr)}) + default: + exe, findErr := findQuickBMS(in.quickbmsPath) + switch { + case errors.Is(findErr, errQuickBMSNotFound): + attempts = append(attempts, sourceAttempt{"QuickBMS auto-extraction", + "no quickbms binary on PATH (install QuickBMS, or set quickbms_path in games.yaml)"}) + case findErr != nil: + // A configured-but-wrong quickbms_path is misconfiguration, not a + // chain-miss: surface it directly rather than burying it in the + // exhausted-chain summary. + return nil, findErr + default: + extracted, exErr := extractWithQuickBMS(ctx, exe, in.basePakPath, cacheDir) + if exErr != nil { + attempts = append(attempts, sourceAttempt{"QuickBMS auto-extraction", exErr.Error()}) + break + } + if vErr := validateDump(extracted, in.basePakPath); vErr != nil { + // The extraction came from the installed pak itself, so a + // validation failure means the extraction was mangled, not that + // it is the wrong week. Do not keep the bad cache. + _ = os.RemoveAll(cacheDir) //nolint:errcheck // best-effort + attempts = append(attempts, sourceAttempt{"QuickBMS auto-extraction", + fmt.Sprintf("extracted tables did not match the installed pak: %v", vErr)}) + break + } + return extracted, nil + } + } + + return nil, chainExhaustedError(attempts) +} + +// chainExhaustedError renders every attempted source, why it failed, and what +// the user can do about it. +func chainExhaustedError(attempts []sourceAttempt) error { + var b strings.Builder + b.WriteString("icarus: no usable source of base data tables for the installed game. Tried:\n") + for _, a := range attempts { + fmt.Fprintf(&b, " - %s: %s\n", a.name, a.reason) + } + b.WriteString("Remedies: point data_dump_path at your own unpacked data.pak JSON tree, " + + "install QuickBMS (or set quickbms_path) so lmm can extract it for you, " + + "or wait for the hosted community dump to catch up with your game version.") + return errors.New(b.String()) +} + +// installRootFromPak recovers the game install root from the base pak's path. +// resolveBasePak builds it as /Icarus/Content/Data/data.pak, so the root +// is four directories up. +func installRootFromPak(basePakPath string) string { + return filepath.Dir(filepath.Dir(filepath.Dir(filepath.Dir(basePakPath)))) +} +``` + +**Migrate every pre-existing call site.** Both signatures changed, so `go build ./...` and +`go vet ./...` are the checklist — but these are the exact sites as of this plan's writing: + +- `newDumpStore` gains a second argument. Eight sites in `datadump_test.go` (lines ~151, 177, + 218, 242, 257, 274, 288, 327), one in `compile_test.go` (~line 30), and one in `icarus.go` + (handled above). In tests, pass `t.TempDir()` as the data dir unless the test asserts on the + extraction cache, in which case hoist it to a variable so the assertion can find it. +- `DumpForBuild` takes the struct. Eight sites in `datadump_test.go` (lines ~154, 180, 221, + 245, 260, 276, 292, 331) and one in `compile.go` (~line 48, further reworked by Task 5): + `store.DumpForBuild(ctx, pak, "")` becomes + `store.DumpForBuild(ctx, chainInput{basePakPath: pak})`, and + `store.DumpForBuild(ctx, pak, local)` becomes + `store.DumpForBuild(ctx, chainInput{basePakPath: pak, localDumpDir: local})`. + +Note what the migration means semantically: those pre-existing tests leave `autoExtract` at its +zero value, `false`, which is correct for them — they were written to exercise the hosted and +local-dir legs, and a stray system-installed QuickBMS must not silently rescue a case they +expect to fail. Any test that _does_ want the auto-run leg sets `autoExtract: true` explicitly +and controls `PATH` (see the new tests above). Leaving the zero value to mean "no auto-run" is +what keeps this migration behavior-preserving. + +Then in `icarus.go`, make `SetDataDir` actually keep the directory — its previous "dataDir's value is unused" comment is now stale and must go: + +```go +// SetDataDir wires the service's data directory into the base-table dump +// store, which uses it for the per-build QuickBMS extraction cache +// (/icarus/extracted//, #174). Compile is gated on this having +// been called at all (see TestIcarus_Compile_WithoutDataDir_FailsLoudly). This +// is a post-construction setter rather than a New parameter because Task 8 of +// the #136 plan froze New(httpClient, projectID) at exactly those two params — +// its call site already depends on that signature — so the data dir arrives the +// same way API keys do: an optional setter the registration pipeline calls when +// present (mirroring its existing SetAPIKey wiring). +func (s *Icarus) SetDataDir(dataDir string) { + s.dumps = newDumpStore(s.firestore.httpClient, dataDir) +} +``` + +Update the stale reference in `cmd/lmm/root.go`'s `registerSource` doc comment, which currently claims dataDir's value is unused: + +```go +// registerSource runs src through the shared registration steps used for +// both built-in and custom sources: collision check (first registration +// wins, warning on customSourceWarnWriter) → API-key resolution (env var via +// envKeyFor, falling back to the stored DB token) → SetAPIKey when the +// source accepts one → SetDataDir when the source accepts one (Icarus's +// Compile is gated on SetDataDir having been called at all: that call +// constructs the base-table dump store, which uses dataDir for its per-build +// QuickBMS extraction cache, #174. New itself can't take dataDir since Task +// 8/9 froze its 2-arg signature) → RegisterSource. +``` + +- [ ] **Step 4: Run tests to verify they pass (GREEN)** + +```bash +go test ./internal/source/icarus/... -v +go build ./... +``` + +Expected: PASS. `go build` catches the `newDumpStore`/`DumpForBuild` call-site migrations. + +- [ ] **Step 5: Commit** + +```bash +git add internal/source/icarus/datadump.go internal/source/icarus/datadump_test.go internal/source/icarus/icarus.go cmd/lmm/root.go +git commit -m "feat: chain base-table sources through a QuickBMS auto-extraction fallback (#174)" +``` + +--- + +## Task 5: Config plumbing — `auto_extract` and `quickbms_path` + +**Files:** + +- Modify: `internal/domain/game.go` (two new `Game` fields) +- Modify: `internal/storage/config/games.go` (two new YAML keys, load + save) +- Modify: `internal/storage/config/games_test.go` +- Modify: `internal/source/source.go` (`Compiler` takes a request struct) +- Modify: `internal/source/icarus/icarus.go`, `internal/source/icarus/compile.go` +- Modify: `internal/core/service.go` (build the request from the game) +- Modify: `internal/core/service_icarus_compile_test.go`, `internal/source/icarus/compile_test.go` + +**Interfaces:** + +- Consumes: `chainInput` (Task 4). +- Produces: `domain.Game.AutoExtract bool`, `domain.Game.QuickBMSPath string`, `source.CompileRequest`, `Compiler.Compile(ctx context.Context, req CompileRequest) error`, `icarus.Compile(ctx context.Context, dumps *DumpStore, req source.CompileRequest) error`. + +> **Design note (resolved ambiguity).** The design says "`Compile`'s exported signature is unchanged", but two new per-game settings have to reach the chain and only the caller holds the game. Keeping positional parameters would make `Compile` take six strings in a row — a transposition hazard flagged during the #136 r4 review. Replacing them with `source.CompileRequest` keeps the _call flow_ unchanged (which is what the design's constraint protects), removes the hazard, and makes the next addition free. No CLI/TUI surface changes. + +- [ ] **Step 1: Write the failing tests** + +Add to `internal/storage/config/games_test.go`: + +```go +func TestLoadGames_AutoExtractDefaultsTrue(t *testing.T) { + dir := t.TempDir() + yaml := "games:\n icarus:\n name: Icarus\n install_path: /games/icarus\n" + + " mod_path: /games/icarus/mods\n" + if err := os.WriteFile(filepath.Join(dir, "games.yaml"), []byte(yaml), 0o644); err != nil { + t.Fatal(err) + } + + games, err := LoadGames(dir) + if err != nil { + t.Fatalf("LoadGames: %v", err) + } + if !games["icarus"].AutoExtract { + t.Error("AutoExtract = false, want true when auto_extract is absent (default true)") + } +} + +func TestLoadGames_AutoExtractExplicitFalse(t *testing.T) { + dir := t.TempDir() + yaml := "games:\n icarus:\n name: Icarus\n install_path: /games/icarus\n" + + " mod_path: /games/icarus/mods\n auto_extract: false\n" + + " quickbms_path: ~/bin/quickbms\n" + if err := os.WriteFile(filepath.Join(dir, "games.yaml"), []byte(yaml), 0o644); err != nil { + t.Fatal(err) + } + + games, err := LoadGames(dir) + if err != nil { + t.Fatalf("LoadGames: %v", err) + } + g := games["icarus"] + if g.AutoExtract { + t.Error("AutoExtract = true, want false when auto_extract: false is set") + } + if g.QuickBMSPath == "" || strings.HasPrefix(g.QuickBMSPath, "~") { + t.Errorf("QuickBMSPath = %q, want a ~-expanded absolute path", g.QuickBMSPath) + } +} + +// auto_extract: false must survive a load/save round trip; the default (true) +// must not be written out, matching deploy_mode's only-if-non-default rule. +func TestSaveGame_AutoExtractRoundTrip(t *testing.T) { + dir := t.TempDir() + for _, g := range []*domain.Game{ + {ID: "icarus", Name: "Icarus", InstallPath: "/g", ModPath: "/m", AutoExtract: false}, + {ID: "other", Name: "Other", InstallPath: "/g2", ModPath: "/m2", AutoExtract: true}, + } { + if err := SaveGame(dir, g); err != nil { + t.Fatalf("SaveGame(%s): %v", g.ID, err) + } + } + raw, err := os.ReadFile(filepath.Join(dir, "games.yaml")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(raw), "auto_extract: false") { + t.Errorf("saved games.yaml should record auto_extract: false, got:\n%s", raw) + } + if strings.Contains(string(raw), "auto_extract: true") { + t.Errorf("saved games.yaml should omit the default auto_extract: true, got:\n%s", raw) + } + + reloaded, err := LoadGames(dir) + if err != nil { + t.Fatalf("LoadGames: %v", err) + } + if reloaded["icarus"].AutoExtract { + t.Error("icarus AutoExtract = true after round trip, want false") + } + if !reloaded["other"].AutoExtract { + t.Error("other AutoExtract = false after round trip, want true") + } +} +``` + +- [ ] **Step 2: Run to verify they fail (RED)** + +```bash +go test ./internal/storage/config/... -run 'AutoExtract' -v +``` + +`games_test.go` currently imports only `os`, `path/filepath` and `testing`; these tests add +`strings` and `github.com/DonovanMods/linux-mod-manager/internal/domain`. + +Expected: FAIL — `domain.Game` has no `AutoExtract`/`QuickBMSPath`. + +- [ ] **Step 3: Implement the config plumbing and request struct** + +In `internal/domain/game.go`, beside `BaseDataPath`: + +```go + // AutoExtract enables extracting the installed data.pak with QuickBMS when + // no other base-table source matches the installed game. Defaults to true; + // set auto_extract: false in games.yaml to opt out (compile games only) + AutoExtract bool + + // QuickBMSPath is optional: an explicit QuickBMS executable for installs + // where it is not on PATH (compile games only) + QuickBMSPath string +``` + +In `internal/storage/config/games.go`, on `GameConfig`: + +```go + AutoExtract *bool `yaml:"auto_extract,omitempty"` + QuickBMSPath string `yaml:"quickbms_path,omitempty"` +``` + +`*bool` rather than `bool` because this setting defaults to **true**: a plain bool cannot distinguish "absent" from "explicitly false", and `omitempty` would drop a legitimate `false`. Pointer-typed optional fields are the existing convention here (`ProfileHookConfigYAML` uses `*string`). + +In `loadGamesLocked`'s `domain.Game` literal, beside `BaseDataPath`: + +```go + AutoExtract: cfg.AutoExtract == nil || *cfg.AutoExtract, + QuickBMSPath: ExpandPath(cfg.QuickBMSPath), +``` + +In `saveGamesLocked`'s `GameConfig` literal, beside `BaseDataPath`: + +```go + QuickBMSPath: game.QuickBMSPath, +``` + +and, next to the existing "only write deploy_mode if not the default" block: + +```go + // Only write auto_extract when it differs from the true default, so a + // hand-written games.yaml stays free of redundant keys. + if !game.AutoExtract { + disabled := false + cfg.AutoExtract = &disabled + } +``` + +In `internal/source/source.go`, replace the `Compiler` interface: + +```go +// CompileRequest carries everything a Compiler needs for one compile. It is a +// struct rather than positional parameters because every field is a string or +// flag resolved from the same game config, and a six-argument call of +// same-typed values is a transposition hazard. +type CompileRequest struct { + // BasePakPath is the installed game's base pak, resolved by the caller + // from game.InstallPath. + BasePakPath string + // BaseDataPath is the game's optional data_dump_path: a local unpacked + // data.pak JSON tree used instead of any other base-table source. "" when + // unset. + BaseDataPath string + // AutoExtract enables the QuickBMS auto-extraction fallback (game's + // auto_extract, default true). + AutoExtract bool + // QuickBMSPath is the game's optional quickbms_path override. "" when + // unset, meaning PATH is searched. + QuickBMSPath string + // SourceFilePath is the just-downloaded file to compile. + SourceFilePath string + // OutputPath is where the compiled result must be written. + OutputPath string +} + +// Compiler is implemented by sources whose downloaded files need +// transforming into a different artifact before deployment (Icarus's +// .exmodz -> .pak). Service consults it, when DeployMode is DeployCompile, +// after downloading but before committing the file to cache — the result +// replaces the downloaded file in cache, so everything downstream (Install, +// the linker) treats it exactly like a DeployCopy file. +type Compiler interface { + Compile(ctx context.Context, req CompileRequest) error +} +``` + +In `internal/source/icarus/icarus.go`: + +```go +// Compile implements source.Compiler by delegating to the package-level +// Compile function. The base-table dump store is supplied from the source +// itself; every per-game setting the chain needs arrives on req, since only +// the caller has the game's config. +func (s *Icarus) Compile(ctx context.Context, req source.CompileRequest) error { + if s.dumps == nil { + return fmt.Errorf("source %q: not initialized with a data directory (SetDataDir was never called)", s.ID()) + } + return Compile(ctx, s.dumps, req) +} +``` + +In `internal/source/icarus/compile.go`, change the signature and the chain call; everything between is untouched: + +```go +// Compile reads req.SourceFilePath's .EXMOD diff, applies it to the game's base +// data tables, bundles in any pre-built assets the .EXMODZ carries, and writes +// the result as a new pak at req.OutputPath ready to deploy as-is. +// +// The base tables come from the source chain in DumpForBuild — a local +// data_dump_path, a cached QuickBMS extraction, the hosted community dump, or a +// fresh QuickBMS auto-run — not from req.BasePakPath: 258 of the 298 tables in +// a real data.pak are Oodle-compressed and cannot be read with the stdlib. +// req.BasePakPath is still opened, for two things it alone can answer — which +// tables the installed game actually has (so a bare, hyphen-flattened +// CurrentFile resolves to a real mount path), and whether the chosen source +// matches the installed week (DumpForBuild byte-checks it against the tables +// the pak stores uncompressed). A source that does not match fails the whole +// compile. +func Compile(ctx context.Context, dumps *DumpStore, req source.CompileRequest) (err error) { + exmodzData, err := os.ReadFile(req.SourceFilePath) + if err != nil { + return fmt.Errorf("icarus: reading %s: %w", req.SourceFilePath, err) + } + bundle, err := ParseExmodz(exmodzData) + if err != nil { + return fmt.Errorf("icarus: %s: %w", req.SourceFilePath, err) + } + + base, err := unrealpak.Open(req.BasePakPath) + if err != nil { + return fmt.Errorf("icarus: opening base pak %s: %w", req.BasePakPath, err) + } + defer base.Close() //nolint:errcheck + + // Loaded and validated before anything is written, so a week mismatch or an + // exhausted source chain fails before a half-built pak exists on disk. + dump, err := dumps.DumpForBuild(ctx, chainInput{ + basePakPath: req.BasePakPath, + localDumpDir: req.BaseDataPath, + autoExtract: req.AutoExtract, + quickbmsPath: req.QuickBMSPath, + }) + if err != nil { + return err + } + + out, err := unrealpak.Create(req.OutputPath) + if err != nil { + return fmt.Errorf("icarus: creating %s: %w", req.OutputPath, err) + } +``` + +The rest of `Compile`'s body is unchanged except that the three remaining `outputPakPath` references inside the cleanup `defer` and the final `out.Close()` error become `req.OutputPath`: + +```go + defer func() { + if err == nil { + return + } + _ = out.Close() //nolint:errcheck + if rmErr := os.Remove(req.OutputPath); rmErr != nil && !os.IsNotExist(rmErr) { + err = fmt.Errorf("%w (additionally, removing partial output %s failed: %v)", err, req.OutputPath, rmErr) + } + }() +``` + +```go + if err := out.Close(); err != nil { + return fmt.Errorf("icarus: finalizing %s: %w", req.OutputPath, err) + } + return nil +} +``` + +`compile.go` gains `"github.com/DonovanMods/linux-mod-manager/internal/source"` in its imports. + +In `internal/core/service.go`, at the compile call site: + +```go + if err := compiler.Compile(ctx, source.CompileRequest{ + BasePakPath: basePakPath, + BaseDataPath: game.BaseDataPath, + AutoExtract: game.AutoExtract, + QuickBMSPath: game.QuickBMSPath, + SourceFilePath: archivePath, + OutputPath: destPath, + }); err != nil { + return nil, fmt.Errorf("compiling mod: %w", err) + } +``` + +Update the `fakeCompilerSource` in `internal/core/service_icarus_compile_test.go` to the new signature: + +```go +func (s *fakeCompilerSource) Compile(ctx context.Context, req source.CompileRequest) error { + s.compileCalls++ + data, err := os.ReadFile(req.SourceFilePath) + if err != nil { + return err + } + return os.WriteFile(req.OutputPath, data, 0o644) +} +``` + +and migrate `internal/source/icarus/compile_test.go`'s `Compile(...)` calls to the struct form, e.g.: + +```go + err := Compile(context.Background(), dumps, source.CompileRequest{ + BasePakPath: basePak, + SourceFilePath: exmodzPath, + OutputPath: outputPath, + }) +``` + +- [ ] **Step 4: Run tests to verify they pass (GREEN)** + +```bash +go build ./... +go test ./internal/storage/config/... ./internal/source/... ./internal/core/... -v 2>&1 | tail -30 +``` + +Expected: PASS across all three packages. + +- [ ] **Step 5: Commit** + +```bash +git add internal/domain/game.go internal/storage/config/games.go internal/storage/config/games_test.go \ + internal/source/source.go internal/source/icarus/icarus.go internal/source/icarus/compile.go \ + internal/source/icarus/compile_test.go internal/core/service.go internal/core/service_icarus_compile_test.go +git commit -m "feat: add auto_extract and quickbms_path game settings (#174)" +``` + +--- + +## Task 6: Announce the auto-run through the existing diagnostics convention + +**Files:** + +- Modify: `internal/source/icarus/datadump.go` +- Modify: `internal/source/icarus/datadump_test.go` + +**Interfaces:** + +- Consumes: `DumpStore` (Task 4). +- Produces: `func (s *DumpStore) SetAnnounceWriter(w io.Writer)`, `func (s *DumpStore) announcef(format string, args ...any)`. + +> **Design note (resolved ambiguity).** The design says the auto-run is "announced through existing progress/logging". This codebase has two such mechanisms and neither fits directly: `ProgressFunc` is download-specific and never reaches `Compile`, and the `Notes`/`Warnings` result-slice convention (`internal/core/flows.go`) requires a result struct that `Compiler.Compile` (error-only) does not have. Threading notes up through `DownloadModResult` would ripple into both the CLI and the TUI — which the "no CLI/TUI surface" constraint forbids. The remaining precedent is `cmd/lmm/root.go`'s `customSourceWarnWriter`: an injectable `io.Writer` defaulting to stderr, used exactly for "deep code must tell the user something". This task follows that, scoped to the store so tests stay parallel-safe (no package-level mutable state). + +- [ ] **Step 1: Write the failing test** + +```go +func TestDumpForBuild_AnnouncesTheAutoRun(t *testing.T) { + skipIfNoShell(t) + const rel = "Factions/D_Factions.json" + body := []byte("{\r\n \"Rows\": []\r\n}") + _, pak := installWithPak(t, map[string][]byte{rel: body}) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(tarGz(t, "IcarusData-old", map[string]string{rel: "{\n \"Rows\": [1]\n}"})) + })) + defer srv.Close() + + binDir := t.TempDir() + stub := writeStubQuickBMS(t, binDir, stubEmitting(map[string]string{ + rel: "{\r\n \"Rows\": []\r\n}", + rootMarkerTable: "{}", + })) + prependToPATH(t, binDir) + + var announced bytes.Buffer + store := newTestStore(t, srv.URL, t.TempDir()) + store.SetAnnounceWriter(&announced) + + if _, err := store.DumpForBuild(context.Background(), + chainInput{basePakPath: pak, autoExtract: true}); err != nil { + t.Fatalf("DumpForBuild: %v", err) + } + + got := announced.String() + if !strings.Contains(got, stub) { + t.Errorf("announcement %q should name the command being run (%s)", got, stub) + } + if !strings.Contains(got, "data.pak") { + t.Errorf("announcement %q should name what is being extracted", got) + } + // The reason matters as much as the action: the user needs to know why an + // external tool suddenly ran. + if !strings.Contains(strings.ToLower(got), "hosted") { + t.Errorf("announcement %q should explain why the fallback was needed", got) + } +} + +func TestDumpStore_AnnounceWriter_DefaultsToStderrAndNeverPanics(t *testing.T) { + s := newDumpStore(http.DefaultClient, t.TempDir()) + // No SetAnnounceWriter call: announcef must be safe on a zero-value writer. + s.announcef("hello %s", "world") +} +``` + +- [ ] **Step 2: Run to verify it fails (RED)** + +```bash +go test ./internal/source/icarus/... -run 'Announce' -v +``` + +Expected: FAIL — `SetAnnounceWriter`, `announcef` undefined. + +- [ ] **Step 3: Implement the announcement** + +Add `"io"` to `datadump.go`'s imports, add the field to `DumpStore`: + +```go +type DumpStore struct { + httpClient *http.Client + dataDir string + treeURL string // overridable in tests + announce io.Writer // nil means os.Stderr; see announcef +} +``` + +and add: + +```go +// SetAnnounceWriter redirects the store's user-facing announcements. Tests +// inject a buffer; production leaves it nil, which means stderr. +func (s *DumpStore) SetAnnounceWriter(w io.Writer) { s.announce = w } + +// announcef tells the user about something they did not ask for directly — +// specifically, that lmm is about to invoke an external tool on their behalf. +// It follows cmd/lmm/root.go's customSourceWarnWriter precedent (an injectable +// writer defaulting to stderr) because Compiler.Compile returns only an error, +// with no result struct to carry the Notes/Warnings slices internal/core uses. +func (s *DumpStore) announcef(format string, args ...any) { + w := s.announce + if w == nil { + w = os.Stderr + } + fmt.Fprintf(w, format, args...) //nolint:errcheck // diagnostics: a failed write must not fail a compile +} +``` + +In `DumpForBuild`'s leg 4, immediately before calling `extractWithQuickBMS`: + +```go + s.announcef("lmm: the hosted base-table dump does not match your installed Icarus "+ + "build (%s), so lmm is extracting the game's own data.pak with QuickBMS.\n"+ + " Running: %s %s\n"+ + " This runs once per game update; results are cached in %s.\n"+ + " Set auto_extract: false in games.yaml to disable this.\n", + build, exe, embeddedScriptName, in.basePakPath, cacheDir) + extracted, exErr := extractWithQuickBMS(ctx, exe, in.basePakPath, cacheDir) +``` + +- [ ] **Step 4: Run tests to verify they pass (GREEN)** + +```bash +go test ./internal/source/icarus/... -v +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add internal/source/icarus/datadump.go internal/source/icarus/datadump_test.go +git commit -m "feat: announce QuickBMS auto-extraction before running it (#174)" +``` + +--- + +## Task 7: Documentation — README, configuration.md, CHANGELOG + +**Files:** + +- Modify: `README.md` +- Modify: `docs/configuration.md` +- Modify: `CHANGELOG.md` + +**Interfaces:** + +- Consumes: the settings from Task 5. +- Produces: no code. + +- [ ] **Step 1: Update `docs/configuration.md`** + +Add two rows to the per-game settings table, beside the existing `data_dump_path` row: + +```markdown +| `auto_extract` | bool | no | Compile-mode only: extract the installed `data.pak` with QuickBMS when no other base-table source matches the installed game (default `true`) | +| `quickbms_path` | string | no | Compile-mode only: explicit QuickBMS executable, for installs where it is not on `PATH` | +``` + +and extend the `compile` deploy-mode paragraph: + +```markdown +- **`compile`**: The downloaded file is compiled into a new artifact before caching (currently Icarus only: an `.exmodz` diff is applied to the game's base data tables to produce a deployable `_P.pak`). Only sources that implement compiling support this mode. Base data tables are resolved in order: your own `data_dump_path` tree, a previously cached QuickBMS extraction for the installed build, the hosted community dump, and finally a fresh QuickBMS extraction of the installed `data.pak`. Every source is byte-validated against the installed game, and a mismatch is a hard error rather than a silent fallback. QuickBMS is optional — if it is not installed, that step is simply skipped — and `auto_extract: false` disables it entirely. Extraction runs once per game update; its output is cached under the lmm data directory. +``` + +- [ ] **Step 2: Update `README.md`** + +Extend the `icarus` example block's commented settings: + +```yaml +icarus: + name: "Icarus" + install_path: "/path/to/Steam/steamapps/common/Icarus" + mod_path: "/path/to/Steam/steamapps/common/Icarus/Icarus/Content/Paks/mods" + deploy_mode: compile + # data_dump_path: ~/icarus-data-dump # Optional: compile from your own + # unpacked data.pak JSON tree instead of the hosted community dump + # auto_extract: false # Optional: disable extracting data.pak with + # QuickBMS when the hosted dump lags your game version (default: true) + # quickbms_path: ~/.local/bin/quickbms # Optional: QuickBMS not on PATH + sources: + icarus: "icarus" +``` + +and add a short paragraph after the block: + +```markdown +Compiling needs the game's base data tables. lmm prefers sources that need no extra tooling — your own `data_dump_path`, then the hosted community dump — but that dump can lag a fresh game update. When it does, and [QuickBMS](https://aluigi.altervista.org/quickbms.htm) is available on `PATH` (or at `quickbms_path`), lmm extracts the installed `data.pak` itself, announces that it is doing so, and caches the result until the next game update. QuickBMS is entirely optional: without it, lmm simply reports that no base-table source matched and tells you the remedies. +``` + +- [ ] **Step 3: Update `CHANGELOG.md`** + +Under `[Unreleased]` → `### Added` (this story adds no version bump; the epic carries one at release): + +```markdown +- **QuickBMS auto-extraction fallback for Icarus compiles**: when the hosted community base-table dump does not match your installed game version, lmm now extracts the game's own `data.pak` using [QuickBMS](https://aluigi.altervista.org/quickbms.htm) — the always-week-correct source — instead of failing. Base tables are resolved in order (`data_dump_path` → cached extraction → hosted dump → fresh extraction), every source is byte-validated against the installed pak, and extraction results are cached per game build so it runs once per update. QuickBMS is optional and runtime-detected: lmm announces the exact command before running it, and two new `games.yaml` settings (`auto_extract`, default `true`, and `quickbms_path`) control it. No new CLI flag or TUI screen (#174) +``` + +- [ ] **Step 4: Verify the docs build/lint clean** + +```bash +trunk check --filter=markdownlint README.md docs/configuration.md CHANGELOG.md 2>&1 | tail -20 +``` + +Expected: no new findings. + +- [ ] **Step 5: Commit** + +```bash +git add README.md docs/configuration.md CHANGELOG.md +git commit -m "docs: document the QuickBMS auto-extraction fallback (#174)" +``` + +--- + +## Task 8: Update the post-plan manual-validation checklist + +**Files:** + +- Modify: `docs/plans/2026-07-29-icarus-exmod-pak-compilation.md` (the #136 plan's "Post-plan manual validation" section — gitignored, so this is a working-doc edit with no commit) + +**Interfaces:** + +- Consumes: everything above. +- Produces: no code. + +- [ ] **Step 1: Add the QuickBMS validation items** + +Append to that plan's "Post-plan manual validation" list: + +```markdown +6. **QuickBMS auto-extraction (#174).** On the reference machine, with the hosted dump still + behind the installed build: + - `quickbms` resolves (Task 1 installed it to `~/.local/bin`); `lmm install` of an Icarus + `.exmodz` prints the announcement naming the command, the pak, and the reason, then + completes. + - The extraction cache appears at `/icarus/extracted//` and holds ~298 + `.json` tables. + - A second install of the same or another `.exmodz` does **not** re-run QuickBMS (the + announcement is absent) — the per-build cache is reused. + - `auto_extract: false` in `games.yaml` restores the previous behavior: the compile fails + with the exhausted-chain error naming all four sources and their remedies. + - Renaming the binary out of `PATH` (with `auto_extract` back to default) produces the + same exhausted-chain error, with the QuickBMS leg reported as "no quickbms binary". +7. **The first real end-to-end compile.** This feature is what finally unblocks it: with a + validated base-table source available, compile the real `Bear_Mount.EXMODZ` from the + catalog and confirm `Bear_Mount_P.pak` is produced, that `unrealpak.Open` on it enumerates + the patched tables plus the bundled assets, and that the patched table differs from the + base table only in the rows the `.EXMOD` targets. Then deploy it and confirm Icarus loads + it in-game — the last unverified link (see item 3's mount-point question, which this is + the natural moment to settle). +``` + +- [ ] **Step 2: No commit** + +`docs/plans/*` is gitignored (`.gitignore:65`). Leave the edit in the working tree. + +--- + +## Post-plan manual validation (this plan) + +1. Run the full suite with no QuickBMS on `PATH` at all — every test in + `internal/source/icarus` must still pass, proving the tool is genuinely optional and that + no test silently depends on a real binary: + + ```bash + env PATH=/usr/bin:/bin go test ./internal/source/icarus/... -count=1 + ``` + +2. Confirm the embedded script actually ships in the binary (a `go:embed` typo fails at build + time, but an empty file does not): + + ```bash + go build -o /tmp/lmm ./cmd/lmm && strings /tmp/lmm | grep -c 'quickbms\|comtype' + ``` + +3. Re-grep for leftover markers once Task 1's answers are folded in: + + ```bash + grep -rn 'SPIKE-CONFIRM' internal/ docs/plans/2026-08-01-icarus-quickbms-fallback.md + ``` + + Every remaining hit must be either resolved-and-deleted or still genuinely unknown. diff --git a/docs/plans/archive/2026-08-01-icarus-zlib-pivot.md b/docs/plans/archive/2026-08-01-icarus-zlib-pivot.md new file mode 100644 index 0000000..2bfd7ce --- /dev/null +++ b/docs/plans/archive/2026-08-01-icarus-zlib-pivot.md @@ -0,0 +1,1409 @@ +# Icarus Zlib Pivot Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Compile Icarus mods against base data tables read directly from the installed game's own `data.pak` — always week-correct, fully offline — by teaching `internal/unrealpak` to decompress Zlib entries, and delete the base-table dump subsystem that only ever existed to route around a blocker that was never there. + +**Architecture:** `unrealpak.Reader` gains two things: the pak's own footer `CompressionMethods` table (previously parsed and discarded), and a Zlib read path in `ReadFile` that reassembles an entry from its per-block deflate streams. `icarus.Compile` then reads base tables straight from the base pak it already opens, which removes its only reason to consult a dump — so the whole dump subsystem (`datadump.go`, the hosted fetch, `validateDump`, `data_dump_path` config plumbing, and the `SetDataDir` wiring that existed to construct the store) is deleted rather than maintained. Oodle and any other unknown method stay a loud `ErrUnsupportedFormat`. + +**Tech Stack:** Go 1.25.6 (this repo's version), standard library only — `compress/zlib` is stdlib, so this adds no dependency. No new third-party packages; `go.mod` is untouched. + +## Why this supersedes #174 + +The Task-1 spike for #174 ([`icarus-quickbms-spike-findings.md`](icarus-quickbms-spike-findings.md)) established two facts by direct measurement: + +1. **`Content/Data/data.pak` contains no Oodle.** Its footer declares `CompressionMethods = ["Zlib"]`, so its 258 compressed tables are Zlib. All 298 tables (40 stored + 258 Zlib) were reconstructed byte-for-byte with stdlib primitives alone — 40,908,881 bytes of decompressed table data, `Items/D_ItemsStatic.json` at 7,304,687 bytes. +2. **The earlier "258 Oodle" reading was a mislabel**, not a measurement error: the #136 rev3 sweep resolved this pak's compression-method _indices_ against `pakchunk0`'s table (`["Oodle","Zlib"]`, index 1 = Oodle) instead of `data.pak`'s own (`["Zlib"]`, index 1 = Zlib). + +That single mislabel is the root of the hosted-dump strategy, the local-override hybrid, and the QuickBMS fallback. With it corrected, all three become unnecessary. [`2026-08-01-icarus-quickbms-fallback.md`](2026-08-01-icarus-quickbms-fallback.md) is obsolete and kept only for history — do not implement it. + +Oodle is real in Icarus, but only in the `Content/Paks/pakchunk0*` **asset** chunks, which hold zero `.json` and which the compile path never reads. + +## Global Constraints + +- **Standard library only.** `compress/zlib` is stdlib; `go.mod` gains nothing. This holds the same line as the rest of the repo (`~/.claude/GO.md`). +- **Fail loud, no silent fallbacks** (repo precedent #95). An unknown or unsupported compression method, a hash mismatch, a block table that doesn't fit its payload, or a base table missing from the installed pak are each a hard error naming what went wrong. Nothing degrades quietly. +- **Method indices are resolved against the pak's own footer table, never assumed.** This is the specific mistake #175 exists to correct; a reader that hardcodes "index 1 = Oodle" is wrong for `data.pak` and a reader that hardcodes "index 1 = Zlib" is wrong for `pakchunk0`. +- **Story branch `dyoung522/175-zlib-pivot`, targeting `epic/icarus-136`** (`--base epic/icarus-136`). Reference **#175** in every commit and in the PR. No version bump in this story — the epic carries one at release. +- **No CLI or TUI surface changes.** The only user-visible change is the _removal_ of the `data_dump_path` game setting from the docs; no flag, command, or screen is added or altered. CLI/TUI parity holds trivially — shared core path. +- **Compiling becomes fully offline.** The "compile requires network access" constraint from the #136 epic no longer applies and is removed with the dump subsystem. + +## Verification status of this plan's code + +Every Go change in Tasks 1–4 was extracted into a scratch copy of the epic branch at `49f1784`, compiled, and run before this plan was finalized: + +```text +gofmt -l ./cmd ./internal -> clean +go build ./... -> OK +go vet ./... -> clean +go test ./... -> 19/19 packages ok, 0 failures (with the dump tests deleted) +grep -rn 'SetDataDir|DumpStore|data_dump_path|BaseDataPath' --include='*.go' . -> 0 hits +``` + +Against the real install: + +```text +data.pak : 298 entries, ReadFile succeeded on 298/298, all valid JSON, sizes match the index + Items/D_ItemsStatic.json = 7,304,687 bytes +pakchunk0 : 5,155 entries now readable (stored + Zlib), 4,138 Oodle refused with + `unsupported pak feature: compression method "Oodle" (index 1)`, 0 unexpected errors +``` + +--- + +## Task 1: `unrealpak` — parse the footer's CompressionMethods table + +**Files:** + +- Modify: `internal/unrealpak/pak.go` +- Modify: `internal/unrealpak/reader.go` +- Create: `internal/unrealpak/zlib_test.go` + +**Interfaces:** + +- Consumes: the existing `readFooter`/`footer` shape. +- Produces: `const maxCompressionMethods`, `const compressionMethodNameSize`, `const compressionMethodsOffset`, `const zlibMethodName`, `const maxUncompressedEntrySize`, `footer.methods [maxCompressionMethods]string`, `Reader.methods`, `func (r *Reader) methodName(method int32) string`. Task 2 depends on all of these. + +- [ ] **Step 1: Write the failing test** + +Create `internal/unrealpak/zlib_test.go` with just the fixture builder and the method-resolution test; Task 2 appends the rest. The Writer only ever emits stored entries, so a compressed fixture has to be hand-assembled. + +```go +package unrealpak + +import ( + "bytes" + "compress/zlib" + "crypto/sha1" //nolint:gosec // pak format uses SHA1, not our choice + "encoding/binary" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +// zlibFixture describes one compressed entry to place in a synthetic pak. +type zlibFixture struct { + path string + blocks [][]byte // each block's PLAINTEXT; each is deflated independently + method int32 // 1-based index into the methods table below +} + +// writeMethodPak hand-builds a version-11 pak whose footer declares methods and +// which holds one compressed entry per fixture. +// +// The Writer only ever emits stored entries, so a compressed fixture has to be +// assembled here. The layout mirrors what a real cooked pak contains: a +// per-entry header carrying the block table, then the deflated blocks, then the +// three index structures and the 221-byte footer. +func writeMethodPak(t *testing.T, methods []string, fixtures []zlibFixture) string { + t.Helper() + const seed uint64 = 0x0123456789ABCDEF + + var data bytes.Buffer + var encoded bytes.Buffer + locations := make(map[string]int32, len(fixtures)) + + for _, fx := range fixtures { + var payload bytes.Buffer + type span struct{ start, end int64 } + hdrSize := compressedHeaderSize(len(fx.blocks)) + var spans []span + var uncompressed int64 + for _, plain := range fx.blocks { + var zbuf bytes.Buffer + zw := zlib.NewWriter(&zbuf) + if _, err := zw.Write(plain); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + start := hdrSize + int64(payload.Len()) + payload.Write(zbuf.Bytes()) + spans = append(spans, span{start, hdrSize + int64(payload.Len())}) + uncompressed += int64(len(plain)) + } + size := int64(payload.Len()) + sum := sha1.Sum(payload.Bytes()) //nolint:gosec + + entryOffset := int64(data.Len()) + var hdr bytes.Buffer + binary.Write(&hdr, binary.LittleEndian, int64(0)) //nolint:errcheck // Offset + binary.Write(&hdr, binary.LittleEndian, size) //nolint:errcheck + binary.Write(&hdr, binary.LittleEndian, uncompressed) //nolint:errcheck + binary.Write(&hdr, binary.LittleEndian, fx.method) //nolint:errcheck + hdr.Write(sum[:]) + binary.Write(&hdr, binary.LittleEndian, int32(len(fx.blocks))) //nolint:errcheck + for _, sp := range spans { + binary.Write(&hdr, binary.LittleEndian, sp.start) //nolint:errcheck + binary.Write(&hdr, binary.LittleEndian, sp.end) //nolint:errcheck + } + hdr.WriteByte(0) // Flags + binary.Write(&hdr, binary.LittleEndian, uint32(65536)) //nolint:errcheck // CompressionBlockSize + if int64(hdr.Len()) != hdrSize { + t.Fatalf("fixture header is %d bytes, compressedHeaderSize says %d", hdr.Len(), hdrSize) + } + data.Write(hdr.Bytes()) + data.Write(payload.Bytes()) + + locations[fx.path] = int32(encoded.Len()) + flags := uint32(1<<31) | uint32(1<<30) | uint32(1<<29) | + uint32(fx.method)<<23 | uint32(len(fx.blocks))<<6 | uint32(65536>>11) + binary.Write(&encoded, binary.LittleEndian, flags) //nolint:errcheck + binary.Write(&encoded, binary.LittleEndian, uint32(entryOffset)) //nolint:errcheck + binary.Write(&encoded, binary.LittleEndian, uint32(uncompressed)) //nolint:errcheck + binary.Write(&encoded, binary.LittleEndian, uint32(size)) //nolint:errcheck + if len(fx.blocks) > 1 { + for _, sp := range spans { + binary.Write(&encoded, binary.LittleEndian, uint32(sp.end-sp.start)) //nolint:errcheck + } + } + } + + // Full directory index: one directory per fixture path. + var fdi bytes.Buffer + binary.Write(&fdi, binary.LittleEndian, int32(len(fixtures))) //nolint:errcheck + for _, fx := range fixtures { + dir, file := splitMountPath(fx.path) + writeFString(&fdi, dir) + binary.Write(&fdi, binary.LittleEndian, int32(1)) //nolint:errcheck + writeFString(&fdi, file) + binary.Write(&fdi, binary.LittleEndian, locations[fx.path]) //nolint:errcheck + } + // Path-hash index, then an empty pruned directory index. + var phi bytes.Buffer + binary.Write(&phi, binary.LittleEndian, int32(len(fixtures))) //nolint:errcheck + for _, fx := range fixtures { + binary.Write(&phi, binary.LittleEndian, hashPath(fx.path, seed)) //nolint:errcheck + binary.Write(&phi, binary.LittleEndian, locations[fx.path]) //nolint:errcheck + } + binary.Write(&phi, binary.LittleEndian, int32(0)) //nolint:errcheck + + phiHash := sha1.Sum(phi.Bytes()) //nolint:gosec + fdiHash := sha1.Sum(fdi.Bytes()) //nolint:gosec + count := int32(len(fixtures)) + indexOffset := int64(data.Len()) + sizing := buildPrimaryIndex(count, seed, 0, 0, phiHash, 0, 0, fdiHash, encoded.Bytes()) + phiOffset := indexOffset + int64(len(sizing)) + fdiOffset := phiOffset + int64(phi.Len()) + index := buildPrimaryIndex(count, seed, phiOffset, int64(phi.Len()), phiHash, + fdiOffset, int64(fdi.Len()), fdiHash, encoded.Bytes()) + indexHash := sha1.Sum(index) //nolint:gosec + + footer := buildFooter(writeVersion, indexOffset, int64(len(index)), indexHash) + for i, name := range methods { + copy(footer[compressionMethodsOffset+i*compressionMethodNameSize:], name) + } + + var out bytes.Buffer + out.Write(data.Bytes()) + out.Write(index) + out.Write(phi.Bytes()) + out.Write(fdi.Bytes()) + out.Write(footer) + + p := filepath.Join(t.TempDir(), "compressed.pak") + if err := os.WriteFile(p, out.Bytes(), 0o644); err != nil { + t.Fatal(err) + } + return p +} + +// The method table is read from the pak's own footer: the SAME index means +// different things in different paks, so an index naming Oodle must be refused +// even though index 1 means Zlib elsewhere. +func TestReadFile_MethodIndexResolvedAgainstThisPaksTable(t *testing.T) { + body := []byte("{}") + p := writeMethodPak(t, []string{"Oodle", "Zlib"}, []zlibFixture{ + {path: "a/Oodled.json", blocks: [][]byte{body}, method: 1}, + {path: "b/Zlibbed.json", blocks: [][]byte{body}, method: 2}, + }) + + r, err := Open(p) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() //nolint:errcheck + + _, err = r.ReadFile("a/Oodled.json") + if !errors.Is(err, ErrUnsupportedFormat) { + t.Fatalf("Oodle entry: err = %v, want ErrUnsupportedFormat", err) + } + if !strings.Contains(err.Error(), "Oodle") { + t.Errorf("Oodle refusal %q should name the method", err) + } + got, err := r.ReadFile("b/Zlibbed.json") + if err != nil { + t.Fatalf("Zlib entry at index 2: %v", err) + } + if !bytes.Equal(got, body) { + t.Errorf("ReadFile = %q, want %q", got, body) + } +} +``` + +- [ ] **Step 2: Run to verify it fails (RED)** + +```bash +cd /home/dyoung/Projects/orca/workspaces/linux-mod-manager/icarus-136 +go test ./internal/unrealpak/... -run TestReadFile_MethodIndex -v +``` + +Expected: FAIL — `compressedHeaderSize`, `compressionMethodsOffset`, `compressionMethodNameSize` undefined. + +- [ ] **Step 3: Add the constants to `pak.go`** + +Insert directly above `// FileEntry describes one file inside a pak`: + +```go +// The footer's CompressionMethods table: 5 fixed-width, NUL-padded name slots +// starting at byte 61. An entry's CompressionMethodIndex is 1-based into it +// (0 means "stored", naming no slot). +const ( + maxCompressionMethods = 5 + compressionMethodNameSize = 32 + compressionMethodsOffset = 61 +) + +// zlibMethodName is the CompressionMethods entry this package can decompress. +// Matched case-insensitively: the name is free-form text written by whatever +// cooked the pak. +const zlibMethodName = "Zlib" + +// maxUncompressedEntrySize caps a single entry's decompressed size. +// +// This deliberately does NOT reuse validateAllocSize's "cannot exceed the pak +// file's own size" rule, which holds for on-disk regions but is simply false +// for decompressed output: Icarus's Items/D_ItemsStatic.json expands to +// 7,304,687 bytes inside a 2,458,743-byte pak. A fixed ceiling is the right +// shape of bound here — it stops a malicious or corrupt UncompressedSize from +// driving an unbounded allocation without rejecting legitimate compression. +const maxUncompressedEntrySize = 512 << 20 +``` + +- [ ] **Step 4: Parse the table in `reader.go`** + +Add the field to `footer`: + +```go +type footer struct { + version int32 + indexOffset int64 + indexSize int64 + indexHash [20]byte + encryptedIndex bool + methods [maxCompressionMethods]string +} +``` + +Replace `readFooter`'s closing comment and `return` (the block currently beginning `// The trailing CompressionMethods name table is intentionally left`) with: + +```go + // The trailing CompressionMethods table names each compression method this + // pak uses; entries reference them by 1-based index. It MUST be read from + // this pak's own footer rather than assumed: Icarus's data.pak declares + // ["Zlib"] (so index 1 means Zlib) while its pakchunks declare + // ["Oodle","Zlib"] (so index 1 means Oodle). Assuming one pak's table + // applies to another is exactly the mislabel that sent #136 chasing an + // Oodle blocker that data.pak never had. + for i := range ft.methods { + slot := buf[compressionMethodsOffset+i*compressionMethodNameSize : compressionMethodsOffset+(i+1)*compressionMethodNameSize] + ft.methods[i] = string(bytes.TrimRight(slot, "\x00")) + } + return ft, nil +} +``` + +Carry the table onto the `Reader`: + +```go +// Reader provides read access to an unencrypted UE4-range pak. Stored entries +// and Zlib-compressed entries are readable; any other compression method is a +// loud ErrUnsupportedFormat. +type Reader struct { + f *os.File + entries []readerEntry + fileSize int64 // total size of the underlying file, for validateAllocSize + methods [maxCompressionMethods]string // this pak's own CompressionMethods table +} +``` + +and in `Open`'s final return: + +```go + return &Reader{f: f, entries: entries, fileSize: fileSize, methods: ft.methods}, nil +``` + +Add the resolver (Task 2 places it beside `ReadFile`; it can live anywhere in the file): + +```go +// methodName resolves a 1-based CompressionMethodIndex against this pak's own +// footer table. An index with no corresponding name yields "", which no +// supported method matches, so it falls through to the unsupported-format +// error rather than being silently treated as stored. +func (r *Reader) methodName(method int32) string { + if method < 1 || int(method) > len(r.methods) { + return "" + } + return r.methods[method-1] +} +``` + +- [ ] **Step 5: Run — still RED, for the right reason** + +```bash +go test ./internal/unrealpak/... -run TestReadFile_MethodIndex -v +``` + +Expected: still FAIL, now on `compressedHeaderSize` undefined — Task 2 supplies the read path. Confirm the constants and table parsing compile: + +```bash +go build ./internal/unrealpak/ +``` + +Expected: OK. + +- [ ] **Step 6: Commit** + +```bash +git add internal/unrealpak/pak.go internal/unrealpak/reader.go internal/unrealpak/zlib_test.go +git commit -m "feat: parse the pak footer's CompressionMethods table (#175)" +``` + +--- + +## Task 2: `unrealpak` — decompress Zlib entries in `ReadFile` + +**Files:** + +- Modify: `internal/unrealpak/reader.go` +- Modify: `internal/unrealpak/zlib_test.go` + +**Interfaces:** + +- Consumes: everything Task 1 produced. +- Produces: `readerEntry.size`, `readerEntry.blocks`, `func compressedHeaderSize(blocks int) int64`, `func (r *Reader) readStored(...)`, `func (r *Reader) readZlib(...)`, and a `ReadFile` that dispatches on the resolved method name. `ReadFile`'s exported signature is unchanged. + +- [ ] **Step 1: Write the failing tests** + +Append to `internal/unrealpak/zlib_test.go`: + +```go +func TestReadFile_ZlibSingleBlock(t *testing.T) { + body := []byte("{\r\n \"Rows\": [1,2,3]\r\n}") + p := writeMethodPak(t, []string{"Zlib"}, []zlibFixture{ + {path: "Factions/D_Factions.json", blocks: [][]byte{body}, method: 1}, + }) + + r, err := Open(p) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() //nolint:errcheck + + got, err := r.ReadFile("Factions/D_Factions.json") + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if !bytes.Equal(got, body) { + t.Errorf("ReadFile = %q, want %q", got, body) + } + if files := r.Files(); len(files) != 1 || files[0].Size != int64(len(body)) { + t.Errorf("Files() = %+v, want one entry sized %d", files, len(body)) + } +} + +// Multi-block reassembly is the case the block table exists for: the blocks +// must be concatenated in order. +func TestReadFile_ZlibMultiBlock(t *testing.T) { + b1 := bytes.Repeat([]byte("alpha "), 400) + b2 := bytes.Repeat([]byte("beta "), 400) + b3 := []byte("tail") + want := append(append(append([]byte{}, b1...), b2...), b3...) + p := writeMethodPak(t, []string{"Zlib"}, []zlibFixture{ + {path: "Items/D_ItemsStatic.json", blocks: [][]byte{b1, b2, b3}, method: 1}, + }) + + r, err := Open(p) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() //nolint:errcheck + + got, err := r.ReadFile("Items/D_ItemsStatic.json") + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if !bytes.Equal(got, want) { + t.Errorf("ReadFile returned %d bytes, want %d (block reassembly)", len(got), len(want)) + } +} + +// An index with no name in the table is unsupported, never silently stored. +func TestReadFile_UnnamedMethodIndex_IsUnsupported(t *testing.T) { + p := writeMethodPak(t, []string{"Zlib"}, []zlibFixture{ + {path: "x/Y.json", blocks: [][]byte{[]byte("{}")}, method: 3}, + }) + + r, err := Open(p) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() //nolint:errcheck + + if _, err := r.ReadFile("x/Y.json"); !errors.Is(err, ErrUnsupportedFormat) { + t.Fatalf("err = %v, want ErrUnsupportedFormat", err) + } +} + +// A corrupted compressed payload must fail the entry's SHA1 gate. +func TestReadFile_ZlibCorruptPayload_FailsHashGate(t *testing.T) { + p := writeMethodPak(t, []string{"Zlib"}, []zlibFixture{ + {path: "c/D.json", blocks: [][]byte{bytes.Repeat([]byte("x"), 200)}, method: 1}, + }) + raw, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + // Flip a byte inside the first entry's compressed payload (just past its + // single-block header). + raw[compressedHeaderSize(1)+2] ^= 0xFF + if err := os.WriteFile(p, raw, 0o644); err != nil { + t.Fatal(err) + } + + r, err := Open(p) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer r.Close() //nolint:errcheck + + if _, err := r.ReadFile("c/D.json"); err == nil { + t.Fatal("expected an error for a corrupted compressed payload, got nil") + } +} +``` + +- [ ] **Step 2: Run to verify they fail (RED)** + +```bash +go test ./internal/unrealpak/... -run TestReadFile_ -v +``` + +Expected: FAIL — `compressedHeaderSize` undefined. + +- [ ] **Step 3: Capture on-disk size and block count in `decodeEntry`** + +Extend `readerEntry`: + +```go +type readerEntry struct { + FileEntry + offset int64 // absolute offset of the entry's on-disk header + method int32 // CompressionMethodIndex; 0 = stored, else a 1-based index + // into the pak footer's CompressionMethods table. + size int64 // on-disk size: compressed for a compressed entry, and equal + // to FileEntry.Size (the uncompressed size) for a stored one. + blocks int // compression block count; 0 for a stored entry. +} +``` + +In `decodeEntry`, replace the discarded Size read: + +```go + offset := read(flags&(1<<31) != 0) + uncompressed := read(flags&(1<<30) != 0) + size := uncompressed // a stored entry does not serialize Size; it equals UncompressedSize + if method != 0 { + size = read(flags&(1<<29) != 0) + } +``` + +and its return: + +```go + return readerEntry{ + FileEntry: FileEntry{Size: uncompressed}, + offset: offset, + method: method, + size: size, + blocks: blockCount, + }, nil +``` + +- [ ] **Step 4: Replace `ReadFile` with a dispatching version plus the two read paths** + +Add `"compress/zlib"` to `reader.go`'s imports. Replace the whole existing `ReadFile` function with: + +```go +// ReadFile returns the bytes of the entry at mount-relative path. +// +// On-disk entry data is preceded by a full FPakEntry header — 53 bytes for a +// stored entry, plus a block table for a compressed one — and the index's +// offset points at that header, not the payload. The header is re-read and +// cross-checked rather than trusted: its method and sizes must agree with the +// index, and its Hash must match the on-disk payload's SHA1. Real paks satisfy +// all of this (verified across a whole install), so a disagreement means +// corruption or a layout this package misread. +func (r *Reader) ReadFile(path string) ([]byte, error) { + for _, e := range r.entries { + if e.Path != path { + continue + } + if e.method == 0 { + return r.readStored(path, e) + } + name := r.methodName(e.method) + if strings.EqualFold(name, zlibMethodName) { + return r.readZlib(path, e) + } + // Oodle and anything else this package cannot decode stay a hard + // error. Refusing here rather than at index-parse time keeps Files() + // able to enumerate a pak whose entries we cannot all read. + return nil, fmt.Errorf("unrealpak: %s: %w: compression method %q (index %d)", + path, ErrUnsupportedFormat, name, e.method) + } + return nil, fmt.Errorf("unrealpak: %s: %w", path, os.ErrNotExist) +} + +// methodName resolves a 1-based CompressionMethodIndex against this pak's own +// footer table. An index with no corresponding name yields "", which no +// supported method matches, so it falls through to the unsupported-format +// error rather than being silently treated as stored. +func (r *Reader) methodName(method int32) string { + if method < 1 || int(method) > len(r.methods) { + return "" + } + return r.methods[method-1] +} + +// readStored reads an uncompressed entry: a 53-byte header then the payload. +func (r *Reader) readStored(path string, e readerEntry) ([]byte, error) { + hdr := make([]byte, storedHeaderSize) + if _, err := r.f.ReadAt(hdr, e.offset); err != nil { + return nil, fmt.Errorf("unrealpak: %s: reading entry header: %w", path, err) + } + if m := int32(binary.LittleEndian.Uint32(hdr[24:28])); m != 0 { + return nil, fmt.Errorf("unrealpak: %s: %w: compressed entry data (method %d)", + path, ErrUnsupportedFormat, m) + } + if size := int64(binary.LittleEndian.Uint64(hdr[8:16])); size != e.Size { + return nil, fmt.Errorf("unrealpak: %s: entry header size %d disagrees with index size %d", + path, size, e.Size) + } + n, err := validateAllocSize(e.Size, r.fileSize) + if err != nil { + return nil, fmt.Errorf("unrealpak: %s: %w", path, err) + } + buf := make([]byte, n) + if _, err := r.f.ReadAt(buf, e.offset+storedHeaderSize); err != nil { + return nil, fmt.Errorf("unrealpak: reading %s: %w", path, err) + } + if sum := sha1.Sum(buf); !bytes.Equal(sum[:], hdr[28:48]) { //nolint:gosec + return nil, fmt.Errorf("unrealpak: %s: content hash mismatch", path) + } + return buf, nil +} + +// compressedHeaderSize is the on-disk size of a compressed entry's FPakEntry +// header: the 53-byte stored shape plus a BlockCount(4) and a 16-byte +// (CompressedStart, CompressedEnd) pair per block, inserted between Hash and +// Flags. +func compressedHeaderSize(blocks int) int64 { + return storedHeaderSize + 4 + 16*int64(blocks) +} + +// readZlib reads and reassembles a Zlib-compressed entry. +// +// The entry's payload is split into independently-deflated blocks. The +// authoritative block table lives in the entry's own on-disk header as +// (CompressedStart, CompressedEnd) pairs measured from the entry offset — the +// index's optional block-size list is omitted for a lone unencrypted block, so +// it cannot be relied on. The blocks tile the payload region contiguously and +// their lengths sum to Size; the header Hash covers those on-disk (compressed) +// bytes, not the decompressed result. +// +// This procedure was validated by reconstructing all 298 tables of the real +// Icarus data.pak — 40 stored plus 258 Zlib — byte-for-byte. +// See docs/plans/icarus-quickbms-spike-findings.md. +func (r *Reader) readZlib(path string, e readerEntry) ([]byte, error) { + if e.blocks <= 0 { + return nil, fmt.Errorf("unrealpak: %s: %w: compressed entry declares %d compression blocks", + path, ErrUnsupportedFormat, e.blocks) + } + hdrSize := compressedHeaderSize(e.blocks) + hn, err := validateAllocSize(hdrSize, r.fileSize) + if err != nil { + return nil, fmt.Errorf("unrealpak: %s: entry header: %w", path, err) + } + hdr := make([]byte, hn) + if _, err := r.f.ReadAt(hdr, e.offset); err != nil { + return nil, fmt.Errorf("unrealpak: %s: reading entry header: %w", path, err) + } + if m := int32(binary.LittleEndian.Uint32(hdr[24:28])); m != e.method { + return nil, fmt.Errorf("unrealpak: %s: entry header method %d disagrees with index method %d", + path, m, e.method) + } + if size := int64(binary.LittleEndian.Uint64(hdr[8:16])); size != e.size { + return nil, fmt.Errorf("unrealpak: %s: entry header size %d disagrees with index size %d", + path, size, e.size) + } + if usize := int64(binary.LittleEndian.Uint64(hdr[16:24])); usize != e.Size { + return nil, fmt.Errorf("unrealpak: %s: entry header uncompressed size %d disagrees with index size %d", + path, usize, e.Size) + } + if nb := int64(int32(binary.LittleEndian.Uint32(hdr[48:52]))); nb != int64(e.blocks) { + return nil, fmt.Errorf("unrealpak: %s: entry header block count %d disagrees with index count %d", + path, nb, e.blocks) + } + + pn, err := validateAllocSize(e.size, r.fileSize) + if err != nil { + return nil, fmt.Errorf("unrealpak: %s: %w", path, err) + } + payload := make([]byte, pn) + if _, err := r.f.ReadAt(payload, e.offset+hdrSize); err != nil { + return nil, fmt.Errorf("unrealpak: reading %s: %w", path, err) + } + if sum := sha1.Sum(payload); !bytes.Equal(sum[:], hdr[28:48]) { //nolint:gosec + return nil, fmt.Errorf("unrealpak: %s: content hash mismatch", path) + } + + if e.Size < 0 || e.Size > maxUncompressedEntrySize { + return nil, fmt.Errorf("unrealpak: %s: %w: uncompressed size %d exceeds the %d-byte cap", + path, ErrUnsupportedFormat, e.Size, int64(maxUncompressedEntrySize)) + } + out := make([]byte, 0, e.Size) + for i := 0; i < e.blocks; i++ { + start := int64(binary.LittleEndian.Uint64(hdr[52+i*16 : 60+i*16])) + end := int64(binary.LittleEndian.Uint64(hdr[60+i*16 : 68+i*16])) + // Block bounds are relative to the entry offset and must land inside + // the payload region that follows the header. + if start < hdrSize || end < start || end > hdrSize+e.size { + return nil, fmt.Errorf("unrealpak: %s: block %d spans [%d,%d), outside the entry's payload", + path, i, start, end) + } + zr, err := zlib.NewReader(bytes.NewReader(payload[start-hdrSize : end-hdrSize])) + if err != nil { + return nil, fmt.Errorf("unrealpak: %s: block %d: %w", path, i, err) + } + // Read at most one byte more than the declared size still allows, so a + // lying UncompressedSize cannot drive an unbounded read. + remaining := e.Size - int64(len(out)) + chunk, err := io.ReadAll(io.LimitReader(zr, remaining+1)) + zr.Close() //nolint:errcheck // read-only decompressor + if err != nil { + return nil, fmt.Errorf("unrealpak: %s: decompressing block %d: %w", path, i, err) + } + if int64(len(chunk)) > remaining { + return nil, fmt.Errorf("unrealpak: %s: decompressed output exceeds the declared uncompressed size %d", + path, e.Size) + } + out = append(out, chunk...) + } + if int64(len(out)) != e.Size { + return nil, fmt.Errorf("unrealpak: %s: decompressed %d bytes, header declares %d", + path, len(out), e.Size) + } + return out, nil +} +``` + +Delete the now-stale trailing comment on `readerEntry.method` that said non-zero entries "cannot be read". + +- [ ] **Step 5: Run tests to verify they pass (GREEN)** + +```bash +gofmt -l ./internal/unrealpak +go test ./internal/unrealpak/... -v +``` + +Expected: all five `TestReadFile_*` tests pass, plus every pre-existing test in the package. + +- [ ] **Step 6: Sanity-check against the real install (manual, not committed)** + +The synthetic fixture proves internal consistency; the real pak proves the format reading. Write a throwaway `main` under the repo (so `internal/` is importable), run it, then delete it: + +```go +package main + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" +) + +func main() { + p, err := unrealpak.Open("/data/SteamLibrary/steamapps/common/Icarus/Icarus/Content/Data/data.pak") + if err != nil { + panic(err) + } + defer p.Close() //nolint:errcheck + ok, bad, total := 0, 0, 0 + for _, f := range p.Files() { + b, err := p.ReadFile(f.Path) + if err != nil { + bad++ + continue + } + var v any + if json.Unmarshal(bytes.TrimPrefix(b, []byte{0xEF, 0xBB, 0xBF}), &v) != nil || int64(len(b)) != f.Size { + bad++ + continue + } + ok++ + total += len(b) + } + fmt.Printf("read+valid-JSON+size-match=%d failed=%d totalBytes=%d\n", ok, bad, total) + big, _ := p.ReadFile("Items/D_ItemsStatic.json") + fmt.Printf("Items/D_ItemsStatic.json = %d bytes\n", len(big)) +} +``` + +Expected, exactly: + +```text +read+valid-JSON+size-match=298 failed=0 totalBytes=40936840 +Items/D_ItemsStatic.json = 7304687 bytes +``` + +- [ ] **Step 7: Commit** + +```bash +git add internal/unrealpak/reader.go internal/unrealpak/zlib_test.go +git commit -m "feat: decompress Zlib pak entries with the standard library (#175)" +``` + +--- + +## Task 3: `icarus.Compile` reads base tables from the installed pak; delete the dump subsystem + +This is one atomic change: `Compile` losing its dump dependency is what makes the dump subsystem dead, and the subsystem cannot be half-removed and still compile. Everything here lands in a single green commit. + +**Files:** + +- Modify: `internal/source/icarus/compile.go` +- Modify: `internal/source/icarus/compile_test.go` +- Modify: `internal/source/icarus/icarus.go` +- Modify: `internal/source/icarus/icarus_test.go` +- Modify: `internal/source/source.go` +- Modify: `internal/core/service.go` +- Modify: `internal/core/service_icarus_compile_test.go` +- Create: `internal/source/icarus/helpers_test.go` +- **Delete:** `internal/source/icarus/datadump.go` +- **Delete:** `internal/source/icarus/datadump_test.go` + +**Interfaces:** + +- Consumes: `unrealpak.Reader.ReadFile` (Task 2). +- Produces: `func Compile(basePakPath, exmodzPath, outputPakPath string) (err error)` (was `Compile(ctx, dumps, basePakPath, localDumpDir, exmodzPath, outputPakPath)`), `source.Compiler.Compile(ctx context.Context, basePakPath, sourceFilePath, outputPath string) error` (drops `baseDataPath`), and `func writeTestBasePak(t *testing.T, files map[string][]byte) string` relocated to `helpers_test.go`. +- Removes: `Dump`, `DumpStore`, `newDumpStore`, `DumpForBuild`, `loadLocalDump`, `fetchTree`, `validateDump`, `toCRLF`, `summarize`, `Build`, `detectBuild`, `Icarus.dumps`, `Icarus.SetDataDir`. + +- [ ] **Step 1: Update the tests first (RED)** + +`writeTestBasePak` currently lives in `datadump_test.go` but is used six times by `compile_test.go`, so it must move before that file is deleted. Create `internal/source/icarus/helpers_test.go`: + +```go +package icarus + +import ( + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" +) + +// writeTestBasePak builds a stored, unencrypted version-11 pak holding one +// entry per (mount-relative path, content) pair, via the Task 4 Writer. It is +// the shared fixture builder for this package's compile tests. +func writeTestBasePak(t *testing.T, files map[string][]byte) string { + t.Helper() + pakPath := filepath.Join(t.TempDir(), "data.pak") + w, err := unrealpak.Create(pakPath) + if err != nil { + t.Fatalf("creating test base pak: %v", err) + } + for rel, data := range files { + if err := w.AddFile(rel, data); err != nil { + t.Fatalf("AddFile(%q): %v", rel, err) + } + } + if err := w.Close(); err != nil { + t.Fatalf("closing test base pak: %v", err) + } + return pakPath +} +``` + +In `compile_test.go`: delete the `testDumpStore` helper entirely, drop the `"context"`, `"net/http"` and `"net/http/httptest"` imports, remove every `dumps := testDumpStore(...)` line, and rewrite each call as `Compile(basePak, exmodzPath, outputPath)`. + +Then replace `TestCompile_DumpWeekMismatch_FailsBeforeWriting` — a week mismatch is no longer expressible, since there is only one source of base tables — with two tests that pin what actually matters now: + +```go +// The base table Compile patches must come from the installed pak itself — +// that is the whole point of the #175 pivot — so a row's output has to reflect +// the pak's own bytes, not any other source. +func TestCompile_PatchesTheBasePaksOwnTable(t *testing.T) { + basePak := writeTestBasePak(t, map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{"BaseMovementSpeed":200,"OnlyInPak":true}}`), + }) + manifest := `{"name":"X","Rows":[{"CurrentFile":"AI-D_AIGrowth.json","File_Items":[{"Name":"Mount_Bear","BaseMovementSpeed":235}]}]}` + exmodzPath := writeTestExmodzFile(t, manifest, nil) + outputPath := filepath.Join(t.TempDir(), "out.pak") + + if err := Compile(basePak, exmodzPath, outputPath); err != nil { + t.Fatalf("Compile: %v", err) + } + r, err := unrealpak.Open(outputPath) + if err != nil { + t.Fatalf("opening compiled output: %v", err) + } + defer r.Close() //nolint:errcheck + got, err := r.ReadFile("AI/D_AIGrowth.json") + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + // The patched field changed... + if !bytes.Contains(got, []byte(`"BaseMovementSpeed":235`)) { + t.Errorf("patched table = %s, want BaseMovementSpeed 235", got) + } + // ...and a field only the base pak carried survived, proving the base + // content was read from the pak rather than synthesized. + if !bytes.Contains(got, []byte(`"OnlyInPak":true`)) { + t.Errorf("patched table = %s, want the base pak's OnlyInPak field preserved", got) + } +} + +// A CurrentFile with no matching entry in the base pak fails before any output +// pak is written. +func TestCompile_UnknownBaseTable_LeavesNoOutputFile(t *testing.T) { + basePak := writeTestBasePak(t, map[string][]byte{ + "AI/D_AIGrowth.json": []byte(`{"Mount_Bear":{}}`), + }) + manifest := `{"name":"X","Rows":[{"CurrentFile":"AI-D_NotInPak.json","File_Items":[{"Name":"X","V":1}]}]}` + exmodzPath := writeTestExmodzFile(t, manifest, nil) + outputPath := filepath.Join(t.TempDir(), "out.pak") + + if err := Compile(basePak, exmodzPath, outputPath); err == nil { + t.Fatal("expected an error for a CurrentFile absent from the base pak, got nil") + } + if _, statErr := os.Stat(outputPath); statErr == nil { + t.Error("no output pak should exist after a failed compile") + } +} +``` + +In `icarus_test.go`: delete `TestIcarus_Compile_WithoutDataDir_FailsLoudly` and `TestIcarus_SetDataDir_ConstructsDumpStore` (both assert on a store that no longer exists), and drop the now-unused `"strings"` import. + +In `internal/core/service_icarus_compile_test.go`, update the fake: + +```go +func (s *fakeCompilerSource) Compile(ctx context.Context, basePakPath, sourceFilePath, outputPath string) error { +``` + +```bash +go vet ./internal/source/icarus/... ./internal/core/... +``` + +Expected: FAIL — `Compile` still has its old signature. + +- [ ] **Step 2: Rewrite `Compile`** + +In `compile.go`, drop the `"context"` import and replace the doc comment, signature, and the dump lookup: + +```go +// Compile reads exmodzPath's .EXMOD diff, applies it to the game's base data +// tables, bundles in any pre-built assets the .EXMODZ carries, and writes the +// result as a new pak at outputPakPath ready to deploy as-is. +// +// Base tables are read directly out of basePakPath — the installed game's own +// Content/Data/data.pak — so they are always week-correct by construction and +// the whole operation is offline. That pak stores 40 tables uncompressed and +// compresses the other 258 with Zlib, all of which internal/unrealpak reads +// with the standard library (#175). basePakPath is also what resolves a bare, +// hyphen-flattened CurrentFile to a real mount path. +// +// There is no ctx parameter: every step is local file I/O over a ~2 MB pak, +// with no network call and no long-running loop to cancel. The +// source.Compiler interface still takes one, for implementations that need it. +func Compile(basePakPath, exmodzPath, outputPakPath string) (err error) { + exmodzData, err := os.ReadFile(exmodzPath) + if err != nil { + return fmt.Errorf("icarus: reading %s: %w", exmodzPath, err) + } + bundle, err := ParseExmodz(exmodzData) + if err != nil { + return fmt.Errorf("icarus: %s: %w", exmodzPath, err) + } + + base, err := unrealpak.Open(basePakPath) + if err != nil { + return fmt.Errorf("icarus: opening base pak %s: %w", basePakPath, err) + } + defer base.Close() //nolint:errcheck + + out, err := unrealpak.Create(outputPakPath) +``` + +(The `dump, err := dumps.DumpForBuild(...)` block and its comment are deleted outright; `unrealpak.Create` now follows the `base.Close()` defer directly.) + +Inside the row loop, replace the dump lookup with a read from the pak: + +```go + baseData, err := base.ReadFile(mountPath) + if err != nil { + return fmt.Errorf("icarus: reading base data table %s: %w", mountPath, err) + } +``` + +- [ ] **Step 3: Simplify the source and the interface** + +`internal/source/source.go` — drop `baseDataPath`: + +```go +// basePakPath is resolved by the caller from game.InstallPath; sourceFilePath +// is the just-downloaded file; outputPath is where the compiled result must be +// written. +type Compiler interface { + Compile(ctx context.Context, basePakPath, sourceFilePath, outputPath string) error +} +``` + +`internal/source/icarus/icarus.go` — drop the `dumps` field, delete `SetDataDir` entirely, and simplify the method: + +```go +type Icarus struct { + firestore *firestoreClient +} +``` + +```go +// Compile implements source.Compiler by delegating to the package-level +// Compile function. ctx is unused: compiling is pure local file I/O against +// the installed game's own pak (#175), with nothing to cancel. +func (s *Icarus) Compile(_ context.Context, basePakPath, sourceFilePath, outputPath string) error { + return Compile(basePakPath, sourceFilePath, outputPath) +} +``` + +`internal/core/service.go` — the call site: + +```go + if err := compiler.Compile(ctx, basePakPath, archivePath, destPath); err != nil { + return nil, fmt.Errorf("compiling mod: %w", err) + } +``` + +- [ ] **Step 4: Delete the dump subsystem** + +```bash +rm internal/source/icarus/datadump.go internal/source/icarus/datadump_test.go +``` + +That removes, in one stroke: the hosted-tree fetch (`fetchTree`, `defaultDumpTreeURL`, `maxDumpBytes`, `maxTarEntrySize`), `loadLocalDump`, `validateDump`, `toCRLF`, `summarize`, `Dump`/`DumpStore`/`newDumpStore`/`DumpForBuild`, and `Build`/`detectBuild`. + +- [ ] **Step 5: Run tests to verify they pass (GREEN)** + +```bash +gofmt -l ./cmd ./internal +go build ./... +go vet ./... +go test ./internal/source/icarus/... ./internal/core/... ./internal/source/... -v 2>&1 | tail -20 +``` + +Expected: build, vet and gofmt clean; all tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add -A internal/source/icarus internal/source/source.go internal/core/service.go internal/core/service_icarus_compile_test.go +git commit -m "feat: compile Icarus mods from the installed pak, drop the dump subsystem (#175)" +``` + +--- + +## Task 4: Remove the `data_dump_path` config plumbing and `SetDataDir` wiring + +With Task 3 landed, `domain.Game.BaseDataPath` has no reader and no source implements `SetDataDir`, so both are dead weight. + +**Files:** + +- Modify: `internal/domain/game.go` +- Modify: `internal/storage/config/games.go` +- **Delete:** `internal/storage/config/games_test.go` +- Modify: `cmd/lmm/root.go` +- Modify: `cmd/lmm/root_test.go` + +**Interfaces:** + +- Removes: `domain.Game.BaseDataPath`, `config.GameConfig.BaseDataPath` (`yaml:"data_dump_path"`), the `SetDataDir` duck-typed setter call, and the `dataDir` parameter from `registerSources`/`registerSource`/`registerCustomSources`. + +- [ ] **Step 1: Remove the field from the domain and config** + +`internal/domain/game.go` — delete: + +```go + // BaseDataPath is optional: a directory holding an unpacked data.pak JSON + // tree, used instead of fetching the hosted base-table dump (compile games only) + BaseDataPath string +``` + +`internal/storage/config/games.go` — delete the struct field, the load mapping, and the save mapping: + +```go + BaseDataPath string `yaml:"data_dump_path,omitempty"` // from GameConfig + BaseDataPath: ExpandPath(cfg.BaseDataPath), // from loadGamesLocked + BaseDataPath: game.BaseDataPath, // from saveGamesLocked +``` + +Removing the longest field name changes struct-tag alignment, so re-run `gofmt -w internal/storage/config/games.go`. + +`internal/storage/config/games_test.go` contains only `TestLoadGames_DataDumpPath`, which tested exactly this key — delete the file: + +```bash +rm internal/storage/config/games_test.go +``` + +- [ ] **Step 2: Remove the `SetDataDir` wiring** + +In `cmd/lmm/root.go`, delete the setter call: + +```go + if setter, ok := src.(interface{ SetDataDir(string) }); ok { + setter.SetDataDir(dataDir) + } +``` + +and drop the now-unused parameter from all three functions and their call sites: + +```go +func registerSources(svc *core.Service, cfgDir string) { + for _, factory := range builtinSourceFactories { + registerSource(svc, factory()) + } + + registerCustomSources(svc, cfgDir) +} +``` + +```go +func registerSource(svc *core.Service, src source.ModSource) { +``` + +```go +func registerCustomSources(svc *core.Service, cfgDir string) { +``` + +with `registerSource(svc, src)` inside `registerCustomSources`, and `registerSources(svc, cfg.ConfigDir)` at the `initService` call site. + +Two doc comments assert the removed behavior and must go with it. In `registerSources`, cut the trailing sentence so it reads: + +```go +// registerSources registers all available mod sources with the service +// through one ordered pipeline: built-ins first (so the collision rule's +// "first wins" preserves their identity against a same-id custom +// definition), then user-defined sources from /sources/. +``` + +and in `registerSource`, end the pipeline description at the API key: + +```go +// source accepts one → RegisterSource. +``` + +The package-level `dataDir` variable (the `--data` flag) is unrelated and stays. + +In `cmd/lmm/root_test.go`, delete the `recordingDataDirSource` type and `TestRegisterSource_WiresDataDir`, then fix the arity of the remaining calls: `registerSource(svc, mock)`, `registerSources(svc, t.TempDir())`, `registerSources(svc, cfgDir)`, `registerCustomSources(svc, cfgDir)`. + +- [ ] **Step 3: Run the full suite (GREEN)** + +```bash +gofmt -l ./cmd ./internal +go build ./... && go vet ./... +go test ./... 2>&1 | grep -E 'FAIL|^ok' +grep -rn 'SetDataDir\|DumpStore\|data_dump_path\|BaseDataPath' --include='*.go' . +``` + +Expected: gofmt/build/vet clean, **19 packages ok and 0 FAIL**, and the final grep returns **no hits** — the subsystem is gone from Go code entirely. + +- [ ] **Step 4: Commit** + +```bash +git add internal/domain/game.go internal/storage/config/games.go cmd/lmm/root.go cmd/lmm/root_test.go +git rm internal/storage/config/games_test.go +git commit -m "refactor: drop the data_dump_path setting and SetDataDir wiring (#175)" +``` + +--- + +## Task 5: Product docs — README, configuration.md, CHANGELOG + +**Files:** + +- Modify: `README.md` +- Modify: `docs/configuration.md` +- Modify: `CHANGELOG.md` + +**Interfaces:** none — documentation only. + +- [ ] **Step 1: `docs/configuration.md`** + +Delete the `data_dump_path` row from the per-game settings table: + +```markdown +| `data_dump_path` | string | no | Compile-mode only: local unpacked data.pak JSON tree, used instead of the hosted base-table dump | +``` + +Removing the longest cell narrows the column, so re-pad the remaining rows so the table stays aligned. + +Replace the `compile` deploy-mode bullet (currently describing `data_dump_path` and the hosted dump) with: + +```markdown +- **`compile`**: The downloaded file is compiled into a new artifact before caching (currently Icarus only: an `.exmodz` diff is applied to the game's base data tables to produce a deployable `_P.pak`). Only sources that implement compiling support this mode. The base data tables are read directly from the installed game's own `data.pak`, so a compile always matches the installed game version and needs no network access. +``` + +- [ ] **Step 2: `README.md`** + +In the `icarus` games.yaml example, delete the two commented `data_dump_path` lines so the block reads: + +```yaml +icarus: + name: "Icarus" + install_path: "/path/to/Steam/steamapps/common/Icarus" + mod_path: "/path/to/Steam/steamapps/common/Icarus/Icarus/Content/Paks/mods" + deploy_mode: compile + sources: + icarus: "icarus" +``` + +- [ ] **Step 3: `CHANGELOG.md`** + +The `[Unreleased]` Icarus entry currently advertises `data_dump_path` and the hosted dump, neither of which will ship. Replace that entry's final sentence — everything from "An optional per-game `data_dump_path`…" to the end — with: + +```markdown +Base data tables are read directly from the installed game's own `data.pak`, so a compile always matches the installed game version and works entirely offline; `internal/unrealpak` reads both the stored and the Zlib-compressed entries that pak contains, using only the standard library (#136, #175) +``` + +- [ ] **Step 4: Verify** + +```bash +trunk check --filter=markdownlint README.md docs/configuration.md CHANGELOG.md 2>&1 | tail -20 +grep -rn 'data_dump_path' README.md docs/ CHANGELOG.md +``` + +Expected: no new markdownlint findings, and the grep returns nothing outside `docs/plans/` (which is gitignored history). + +- [ ] **Step 5: Commit** + +```bash +git add README.md docs/configuration.md CHANGELOG.md +git commit -m "docs: drop data_dump_path, document compiling from the installed pak (#175)" +``` + +--- + +## Task 6: Correct the #136 findings and plan docs + +These are gitignored in-flight documents (`.gitignore:65 docs/plans/*`), so there is nothing to commit — but they are the record future work reads, and both currently assert an Oodle blocker that does not exist. **Correct them inline with a dated note; do not silently rewrite history.** The wrong conclusion is instructive, and erasing it would hide a real methodological lesson. + +**Files:** + +- Modify: `docs/plans/icarus-pak-format-findings.md` +- Modify: `docs/plans/2026-07-29-icarus-exmod-pak-compilation.md` + +**Interfaces:** none — documentation only. + +- [ ] **Step 1: Correct the findings doc** + +`icarus-pak-format-findings.md` asserts the blocker in three places (§ "⚠ BLOCKING RISK", the Part 2 verdict item 2, and a Part 3 line claiming `data.pak` "contains **no Zlib entries** (only 40 stored + 258 Oodle)"). Prepend a correction banner immediately under the "⚠ BLOCKING RISK: 258 of `data.pak`'s 298 JSON files are Oodle-compressed" heading: + +```markdown +> **CORRECTION (2026-08-01, #175): this section is WRONG — those 258 tables are Zlib, not Oodle.** +> The counts here are right; the method _name_ is not. This section resolved `data.pak`'s +> compression-method indices against `pakchunk0`'s method table (`["Oodle","Zlib"]`, where +> index 1 = Oodle) instead of reading `data.pak`'s own footer table, which is `["Zlib"]` — +> so index 1 means **Zlib** in this pak. All 258 decompress with stdlib `compress/zlib`; +> all 298 tables were reconstructed byte-for-byte. See +> [`icarus-quickbms-spike-findings.md`](icarus-quickbms-spike-findings.md) §6b and plan +> [`2026-08-01-icarus-zlib-pivot.md`](2026-08-01-icarus-zlib-pivot.md). +> +> Everything below is left as originally written. The lesson is worth keeping: a per-pak +> fact was asserted from another pak's metadata, and that single mislabel drove the hosted +> dump strategy, the local-override hybrid, and the QuickBMS fallback — all now obsolete. +``` + +Add a one-line correction to the Part 2 verdict's item 2 and to the Part 3 "no Zlib entries" line, each pointing at the same banner, e.g.: + +```markdown +> **CORRECTED 2026-08-01 (#175):** `data.pak` is Zlib, not Oodle — see the correction banner in Part 3. +``` + +- [ ] **Step 2: Correct the #136 plan doc's prose** + +`2026-07-29-icarus-exmod-pak-compilation.md` asserts the blocker in its Global Constraints (the "compile path requires network access" bullet, which cites Oodle), in Task 12's `> **Base tables come from a hosted per-week dump…** ` note, in Task 12a's rationale, and in Task 13's surrounding prose. Add a single banner immediately below the plan's `# ` title rather than editing each site: + +```markdown +> **SUPERSEDED IN PART (2026-08-01, #175).** Everywhere this plan states that `data.pak`'s +> tables are Oodle-compressed and therefore unreadable — the Global Constraints' network +> bullet, Task 12's base-table note, Task 12a (the dump fetcher) and Task 13's `data_dump_path` +> wiring — the premise is false: those tables are **Zlib**, which the standard library reads. +> Tasks 1–11 (the pak format work, the Firestore source, `.EXMOD`/`.EXMODZ` handling) are +> unaffected and shipped as written. The dump subsystem those later tasks built has been +> removed by [`2026-08-01-icarus-zlib-pivot.md`](2026-08-01-icarus-zlib-pivot.md); read that +> for the current design. This document is kept unedited below as the record of how the epic +> was actually built. +``` + +- [ ] **Step 3: Verify the cross-references resolve** + +```bash +grep -n 'CORRECTION\|CORRECTED\|SUPERSEDED IN PART' docs/plans/icarus-pak-format-findings.md docs/plans/2026-07-29-icarus-exmod-pak-compilation.md +ls docs/plans/icarus-quickbms-spike-findings.md docs/plans/2026-08-01-icarus-zlib-pivot.md +``` + +Expected: four correction markers, and both referenced files exist. + +- [ ] **Step 4: No commit** + +`docs/plans/*` is gitignored. Leave the edits in the working tree and record completion in the task tracker. + +--- + +## Task 7: Manual validation gate — the first real end-to-end compile + +This is the acceptance gate the whole epic has been blocked on, and #175 is what unblocks it. **Not automated, not CI**: it needs the real game install and a real mod. Run it on the reference machine after Tasks 1–5 are green. + +**Files:** none — this task produces a `_P.pak` on disk and a recorded result. + +**Interfaces:** none. + +- [ ] **Step 1: Obtain a real `.EXMODZ`** + +`Bear_Mount.EXMODZ` is the mod the epic's fixtures were modelled on. Fetch it from the catalog the Icarus source already reads, or from the ecosystem mods repo: + +```bash +mkdir -p /tmp/icarus-e2e && cd /tmp/icarus-e2e +curl -sL -o Bear_Mount.EXMODZ \ + "https://github.com/Jimk72/Icarus_Mods/raw/main/Bear_Mount.EXMODZ" +ls -la Bear_Mount.EXMODZ && file Bear_Mount.EXMODZ +``` + +Expected: a multi-megabyte Zip archive (~2.7 MB at the time of writing). If the URL has moved, `lmm search icarus bear` and the catalog's download URL is the supported route — the point is a genuine, unmodified mod file, not a fixture. + +- [ ] **Step 2: Compile it against the real install** + +Write a throwaway `main` inside the repo (so `internal/` is importable), run it, and delete it afterwards: + +```go +package main + +import ( + "fmt" + + "github.com/DonovanMods/linux-mod-manager/internal/source/icarus" +) + +func main() { + const ( + basePak = "/data/SteamLibrary/steamapps/common/Icarus/Icarus/Content/Data/data.pak" + exmodz = "/tmp/icarus-e2e/Bear_Mount.EXMODZ" + out = "/tmp/icarus-e2e/Bear_Mount_P.pak" + ) + if err := icarus.Compile(basePak, exmodz, out); err != nil { + fmt.Println("COMPILE FAILED:", err) + return + } + fmt.Println("COMPILE OK ->", out) +} +``` + +```bash +go run ./cmd/e2e-compile # or wherever the throwaway main was placed +ls -la /tmp/icarus-e2e/Bear_Mount_P.pak +``` + +**Expected outcome:** `COMPILE OK`, and a `Bear_Mount_P.pak` of non-trivial size on disk. This is the first time this pipeline has produced a real artifact — every previous attempt died at the base-table step. + +If it fails, the error names the step. The two most likely genuine failures, neither of which is a Zlib problem: + +- `no matching file in base pak (expected mount path …)` — the `.EXMOD`'s `CurrentFile` does not resolve. Record the exact `CurrentFile` and the mount paths present; that is a `matchMountPath` finding, not a pivot regression. +- A `.EXMOD` schema surprise (a row shape `ApplyRowPatch` does not handle). Record the offending row verbatim. + +- [ ] **Step 3: Inspect the produced pak** + +```go +package main + +import ( + "fmt" + + "github.com/DonovanMods/linux-mod-manager/internal/unrealpak" +) + +func main() { + r, err := unrealpak.Open("/tmp/icarus-e2e/Bear_Mount_P.pak") + if err != nil { + panic(err) + } + defer r.Close() //nolint:errcheck + for _, f := range r.Files() { + fmt.Printf("%10d %s\n", f.Size, f.Path) + } +} +``` + +**Expected outcome:** the output pak enumerates cleanly and lists (a) every data table the mod patched, at a size close to the base table's, and (b) every asset the `.EXMODZ` bundled. Record the full listing. + +- [ ] **Step 4: Confirm the patch actually applied** + +For one patched table, diff the compiled pak's copy against the installed pak's copy — they must differ **only** in the fields the `.EXMOD` targets: + +```bash +# read the same table from both paks via a throwaway main, write to /tmp, then: +diff <(python3 -m json.tool /tmp/icarus-e2e/base_table.json) \ + <(python3 -m json.tool /tmp/icarus-e2e/patched_table.json) | head -40 +``` + +**Expected outcome:** a small, targeted diff on the modded rows only (for Bear_Mount, mount/creature stats), with every other row byte-identical. + +- [ ] **Step 5: Deploy and load in-game (the last unverified link)** + +Copy the `_P.pak` into the game's mod path and launch Icarus. + +**Expected outcome:** the game loads and the mod's effects are visible. If the game loads but the mod has no effect, the most likely cause is the **mount point** — `unrealpak.Writer` stamps `defaultMountPoint` (`"../../../"`), while the real `data.pak` uses an absolute cook path (`C:/BA/work/.../Temp/Data/`). That question has been open since the #136 spike and this is the moment it can finally be settled; record which mount point works and file a follow-up if it needs changing. + +This step is the only one whose outcome is genuinely unknown — Steps 1–4 are expected to pass. Record what happens either way. + +- [ ] **Step 6: Record the result** + +Write the outcome into the epic's tracker: whether a `_P.pak` was produced, its size and entry count, the patched-table diff, and the in-game result. Nothing to commit. + +--- + +## Post-plan verification + +1. Whole suite from a clean cache, to catch anything the incremental runs cached: + + ```bash + go clean -testcache && go test ./... + ``` + + Expected: 19 packages ok, 0 failures. + +2. No trace of the removed subsystem outside gitignored history: + + ```bash + grep -rn 'SetDataDir\|DumpStore\|DumpForBuild\|validateDump\|data_dump_path\|BaseDataPath' \ + --include='*.go' --include='*.md' --include='*.yaml' . | grep -v '^./docs/plans/' + ``` + + Expected: no hits. + +3. The reader still refuses what it cannot read, on the real install: `ReadFile` on any + `pakchunk0` Oodle entry returns `ErrUnsupportedFormat` naming `"Oodle"`, while its stored + and Zlib entries read successfully (5,155 readable / 4,138 refused / 0 unexpected errors at + the time of writing). diff --git a/docs/plans/archive/icarus-pak-format-findings.md b/docs/plans/archive/icarus-pak-format-findings.md new file mode 100644 index 0000000..026810a --- /dev/null +++ b/docs/plans/archive/icarus-pak-format-findings.md @@ -0,0 +1,852 @@ +# Icarus PAK Footer/Index Format — Empirical Findings (#136 Task 1) + +Scratch findings doc from an empirical, read-only spike against the user's real Icarus Steam +install. No product code was written. Not shipped as product docs — kept for Task 2's +reference while implementing `internal/unrealpak`. + +## Files inspected + +Icarus's Steam install lives on a secondary library, not under `~/.steam` or +`~/.local/share/Steam` as the task brief assumed — found instead under `/data/SteamLibrary`. + +Base pakchunk (primary subject of this spike): + +```text +/data/SteamLibrary/steamapps/common/Icarus/Icarus/Content/Paks/pakchunk0-WindowsNoEditor.pak +size: 1,421,136,117 bytes +``` + +Icarus ships one base pakchunk (`pakchunk0-WindowsNoEditor.pak`) plus 32 numbered split +pakchunks (`pakchunk0_s1` .. `pakchunk0_s32`), all in the same directory, ranging from +~47 MB to ~1.9 GB. A second pakchunk was spot-checked for consistency (see Cross-check +section below): + +```text +/data/SteamLibrary/steamapps/common/Icarus/Icarus/Content/Paks/pakchunk0_s10-WindowsNoEditor.pak +size: 1,645,159,864 bytes +``` + +## Result summary + +| Field | Value | +| ----------------- | ---------------------------------------------------------------------------------------------- | +| `Version` | **11** (`PakFile_Version_Fnv64BugFix`) | +| Footer size | **221 bytes** — matches the plan's upper-bound assumption | +| `bEncryptedIndex` | **0 (false)** — confirmed, but the brief's naive parse script reads the wrong byte (see below) | +| SHA1 cross-check | **Match** | + +The plan's core assumptions (221-byte footer, `bEncryptedIndex == false`) are **confirmed**. +However, the brief's Step 3 field-offset script does not correctly locate `bEncryptedIndex` +for this pak version — see "Deviation from the brief's naive layout" below. This is a +significant finding for Task 2's implementation and must be reflected in +`internal/unrealpak`'s field-offset constants. + +## Step 2: Locating the magic + +```bash +PAK=/data/SteamLibrary/steamapps/common/Icarus/Icarus/Content/Paks/pakchunk0-WindowsNoEditor.pak +tail -c 256 "$PAK" | xxd | tail -20 +``` + +``` +00000000: 0100 0000 1700 0000 5669 7375 616c 4465 ........VisualDe +00000010: 6275 6767 6572 2e75 706c 7567 696e 0058 bugger.uplugin.X +00000020: 8002 0000 0000 0000 0000 0000 0000 0000 ................ +00000030: 0000 0000 e112 6f5a 0b00 0000 f6f3 a954 ......oZ.......T +00000040: 0000 0000 d680 0200 0000 0000 5d2f 07c9 ............]/.. +00000050: 209f 8a50 f17e 15a3 f656 2e07 d85b 9ab0 ..P.~...V...[.. +00000060: 4f6f 646c 6500 0000 0000 0000 0000 0000 Oodle........... +00000070: 0000 0000 0000 0000 0000 0000 0000 0000 ................ +00000080: 5a6c 6962 0000 0000 0000 0000 0000 0000 Zlib............ +00000090: 0000 0000 0000 0000 0000 0000 0000 0000 ................ +...(zero-padded to end of file)... +``` + +Magic (`E1 12 6F 5A`, i.e. `0x5A6F12E1` little-endian) found at offset 52 within the +last-256-byte tail window (absolute file offset 1,421,135,913). No larger tail window was +needed — 256 bytes was sufficient. + +## Step 3: Parsing the footer — brief's naive script vs. actual UE4 layout + +Running the brief's Step 3 script verbatim against the found magic offset: + +``` +version: 11 +index_offset: 1420424182 index_size: 164054 +index_hash (hex): 5d2f07c9209f8a50f17e15a3f6562e07d85b9ab0 +footer start offset: 1421135913 -> footer size: 204 +bEncryptedIndex byte: 79 +``` + +`version`, `index_offset`, `index_size`, and `index_hash` all parse correctly (confirmed by +the SHA1 cross-check below). But `bEncryptedIndex byte: 79` (0x4F, ASCII `'O'`) is **not a +valid bool byte** — it's actually the first character of the string `"Oodle"` from the +compression-methods block. The brief's script assumes `bEncryptedIndex` immediately follows +`IndexHash`, which is the pre-UE4.25 layout and does not hold for version 11. + +### Actual `FPakInfo` layout for version 11 (`PakFile_Version_Fnv64BugFix`) + +Per UE4/5 source (`IPlatformFilePak.h`, `FPakInfo::Serialize`), two version gates change the +footer shape between the pre-UE4.22 layout the plan assumed and version 11: + +- `PakFile_Version_EncryptionKeyGuid` (7): adds a 16-byte `EncryptionKeyGuid` **before** + `Magic`. +- `PakFile_Version_IndexEncryption` (4): adds the 1-byte `bEncryptedIndex` **before** `Magic` + (immediately after `EncryptionKeyGuid` when both apply). +- `PakFile_Version_FNameBasedCompressionMethod` (8): adds a `CompressionMethods` array + (5 slots × 32 bytes, fixed-width null-terminated ASCII names) **after** `IndexHash`. + +So the true footer layout for version 11 is: + +``` +EncryptionKeyGuid (16 bytes) +bEncryptedIndex (1 byte) +Magic (4 bytes) +Version (4 bytes, int32) +IndexOffset (8 bytes, int64) +IndexSize (8 bytes, int64) +IndexHash (20 bytes) +CompressionMethods (5 × 32 = 160 bytes) +--- +Total: 16+1+4+4+8+8+20+160 = 221 bytes +``` + +Re-parsing with this corrected layout (offsets relative to the magic offset `off`): + +``` +magic absolute offset: 1421135913 +EncryptionKeyGuid bytes (16, hex): 00000000000000000000000000000000 +bEncryptedIndex byte (correct offset, off-1): 0 +version: 11 +index_offset: 1420424182 index_size: 164054 +index_hash (hex): 5d2f07c9209f8a50f17e15a3f6562e07d85b9ab0 +true footer start (guid_off = off-17): 1421135896 +true footer size (EOF - guid_off): 221 +compression method slot 0: 'Oodle' +compression method slot 1: 'Zlib' +compression method slot 2: '' +compression method slot 3: '' +compression method slot 4: '' +file size: 1421136117 +``` + +`EncryptionKeyGuid` is all-zero (no per-pak encryption key) and `bEncryptedIndex == 0` +(false) — the plan's assumption is confirmed, just at a different byte offset than the +brief's naive script computes. Footer size is **221 bytes**, matching the upper bound of +the plan's `footerSizes` assumption exactly (the 61-byte alternative does not apply here). + +Compression methods declared: `Oodle` and `Zlib` (3 of 5 slots unused/zero-padded). + +## Step 4: SHA1 cross-check + +```bash +python3 -c " +import hashlib +data = open('$PAK', 'rb').read() +index_offset = 1420424182 +index_size = 164054 +index_bytes = data[index_offset:index_offset+index_size] +print(hashlib.sha1(index_bytes).hexdigest()) +" +``` + +``` +sha1 of index bytes: 5d2f07c9209f8a50f17e15a3f6562e07d85b9ab0 +expected (footer index_hash): 5d2f07c9209f8a50f17e15a3f6562e07d85b9ab0 +``` + +**Match.** This is strong confirmation that `Version`, `IndexOffset`, `IndexSize`, and +`IndexHash` parsing (using the magic-relative offsets, unaffected by the +`bEncryptedIndex` discrepancy above) is correct. + +## Cross-check against a second pakchunk + +Spot-checked `pakchunk0_s10-WindowsNoEditor.pak` (1,645,159,864 bytes) using the corrected +layout, to confirm the format is consistent across Icarus's pakchunks and not an artifact +of the base chunk specifically: + +``` +version: 11 footer size: 221 +bEncryptedIndex: 0 +index_hash: 5bab16b081e26f151a7e827edf0092654825cf2c +sha1 computed: 5bab16b081e26f151a7e827edf0092654825cf2c +match: True +``` + +Same version, same footer size, same `bEncryptedIndex == false`, and SHA1 match. High +confidence the format is uniform across all of Icarus's pakchunks. + +## Implications for Task 2 (`internal/unrealpak`) + +1. **Footer size 221 bytes is confirmed** — proceed with that constant for version-11-class + paks (no need to special-case the 61-byte alternative for Icarus). +2. **`bEncryptedIndex == false` is confirmed** — but Task 2's reader must place + `bEncryptedIndex` (and `EncryptionKeyGuid`) **before** `Magic`, not after `IndexHash`, and + must account for the trailing `CompressionMethods` array (5 × 32 bytes) for version ≥ 8. + A reader that reuses the brief's naive Step 3 offsets verbatim will silently read garbage + for `bEncryptedIndex` (it happened to land inside a compression-method name string here) + and would misreport the footer size as 204 instead of 221. +3. Recommend implementing the footer parser as: read the last N bytes, locate `Magic` + by scanning backward from EOF (not by assuming a fixed footer size up front, since the + size varies with version/feature gates), then compute `version`/`index_offset`/ + `index_size`/`index_hash` relative to the magic offset, and separately compute + `bEncryptedIndex`/`EncryptionKeyGuid` relative to the magic offset going _backward_ + (`magic_offset - 1` and `magic_offset - 17` respectively) rather than forward from the + hash. +4. No indication of index encryption or per-pak encryption keys in either sampled pakchunk — + `internal/unrealpak` does not need an encrypted-index code path for Icarus's shipped data + (mod-authoring use case), though it may still be worth a defensive error if + `bEncryptedIndex != 0` is ever encountered. + +## Overall verdict (footer spike) + +**DONE_WITH_CONCERNS** (format substantially confirmed, but the brief's naive parsing script +needs correcting): `Version == 11`, footer size `== 221` bytes (plan's upper-bound +assumption holds), `bEncryptedIndex == false` (confirmed, at a corrected offset), and the +SHA1 cross-check matches exactly on two independent pakchunks. Task 2 should implement the +corrected offset layout described above rather than the brief's Step 3 script verbatim. + +--- + +# Part 2 — Empirical index decode (#136 plan rev2 spike) + +The footer spike above stopped at the footer. This second read-only spike decodes the +**index** itself, because the plan's Task 2 assumed a _classic_ flat index +(`MountPoint`, `NumEntries`, then N inline `FPakEntry` records). **That assumption is +false for version 11.** Version 11 uses the UE 4.25+ three-part index: a primary index +holding _bit-packed_ entry records plus offsets to two secondary indexes (a path-hash +index and a full directory index), each SHA1-gated. + +Everything below was decoded and verified byte-for-byte with `python3` against the real +install. **Every structural claim in Part 2 was verified across all 34 paks on disk** +(the 33 `Content/Paks/pakchunk0*` chunks plus `Content/Data/data.pak`) — 173,078 entries +total — not just a sample. + +## The pak that actually matters: `Content/Data/data.pak` + +The Task 1 brief looked only in `Content/Paks/`. Icarus keeps its **moddable JSON data +tables in a separate pak** that the earlier spike never examined: + +```text +/data/SteamLibrary/steamapps/common/Icarus/Icarus/Content/Data/data.pak +size: 2,458,743 bytes version 11 221-byte footer 298 entries — ALL .json +MountPoint: 'C:/BA/work/92bbbfa44df12262/Temp/Data/' PathHashSeed: 0xfceb4085 +``` + +This is the `.EXMOD` base pak — the one Task 12's `resolveCurrentFile` must open. The +`Content/Paks/pakchunk0*` chunks contain **zero** `.json` files (verified by enumerating +all 33 chunks: 172,780 entries, extensions are `uexp`/`uasset`/`ubulk`/`res`/`umap`/`png`/ +`ini`/…). The plan's `resolveBasePak` must point at `Icarus/Content/Data/data.pak`, not a +pakchunk. + +### ⚠ BLOCKING RISK: 258 of `data.pak`'s 298 JSON files are Oodle-compressed + +> **CORRECTION (2026-08-01, #175): this section is WRONG — those 258 tables are Zlib, not Oodle.** +> The counts here are right; the method _name_ is not. This section resolved `data.pak`'s +> compression-method indices against `pakchunk0`'s method table (`["Oodle","Zlib"]`, where +> index 1 = Oodle) instead of reading `data.pak`'s own footer table, which is `["Zlib"]` — +> so index 1 means **Zlib** in this pak. All 258 decompress with stdlib `compress/zlib`; +> all 298 tables were reconstructed byte-for-byte. See +> [`icarus-quickbms-spike-findings.md`](icarus-quickbms-spike-findings.md) §6b and plan +> [`2026-08-01-icarus-zlib-pivot.md`](2026-08-01-icarus-zlib-pivot.md). +> +> Everything below is left as originally written. The lesson is worth keeping: a per-pak +> fact was asserted from another pak's metadata, and that single mislabel drove the hosted +> dump strategy, the local-override hybrid, and the QuickBMS fallback — all now obsolete. + +| | files | uncompressed bytes | +| -------------------------------------- | ----- | ------------------ | +| `CompressionMethodIndex 0` (stored) | 40 | 27,959 | +| `CompressionMethodIndex 1` (**Oodle**) | 258 | 40,908,881 | + +The 40 stored files are all tiny stubs (113–~1 KB: `D_Factions.json`, `D_LevelSequences.json`, +…). **Every data table a mod would realistically patch is Oodle-compressed** — +`Items/D_ItemsStatic.json` (7.3 MB), `Crafting/D_ProcessorRecipes.json` (3.9 MB), +`Talents/D_Talents.json` (2.6 MB), `Traits/D_Itemable.json` (2.2 MB), `AI/D_AISetup.json` +(1.3 MB), `Quests/D_Quests.json` (1.2 MB). + +Oodle is a proprietary codec with no Go stdlib implementation, so the plan's +"uncompressed-only" **read** path cannot read the files Task 12 needs to patch. This does +not affect the **write** path (our own paks can store everything uncompressed). It is +recorded here and in the plan as an unresolved blocker for Task 12 — it needs a product +decision, not a format fix. + +## Primary index layout (version ≥ 10, `PakFile_Version_PathHashIndex`) + +Located at the footer's `IndexOffset`/`IndexSize`; SHA1 gated by the footer's `IndexHash`. + +```text +FString MountPoint // "../../../" (pakchunk0) / "C:/BA/.../Temp/Data/" (data.pak) +int32 NumEntries +uint64 PathHashSeed +int32 bHasPathHashIndex // 1 + int64 PathHashIndexOffset // absolute file offset + int64 PathHashIndexSize + [20] PathHashIndexHash // SHA1 of that region +int32 bHasFullDirectoryIndex // 1 + int64 FullDirectoryIndexOffset + int64 FullDirectoryIndexSize + [20] FullDirectoryIndexHash // SHA1 of that region +int32 EncodedPakEntriesSize +uint8[] EncodedPakEntries // bit-packed records, see below +int32 NumNonEncodedFiles // 0 in all 34 paks; would be followed by full FPakEntry records +``` + +Decoded from the real `pakchunk0`: + +```text +MountPoint: '../../../' +NumEntries: 9295 +PathHashSeed: 0x000000009c4dd25a +bHasPathHashIndex: 1 off=1420588236 size=201687 sha1=52ce88320eae17b776eea8d6e1344d28a19728ac +bHasFullDirectoryIndex: 1 off=1420789923 size=345973 sha1=9a2878db6f62cf566f591c681fa58ba55973ca1e +EncodedPakEntriesSize: 163940 (blob begins 110 bytes into the primary index) +NumNonEncodedFiles: 0 +bytes remaining after NumNonEncodedFiles: 0 <- primary index consumed exactly +``` + +`FString` is the standard UE encoding: `int32 Len`; `Len > 0` → `Len` ANSI bytes +_including_ a trailing NUL; `Len < 0` → `-Len` UTF-16LE code units including NUL. +All 34 paks use the ANSI form exclusively. + +### Region tiling + +The four trailing regions tile the file exactly, with no gaps and no padding: + +```text +primary [1420424182, 1420588236) size=164054 gap_from_prev=- +path-hash [1420588236, 1420789923) size=201687 gap_from_prev=0 +full-dir [1420789923, 1421135896) size=345973 gap_from_prev=0 +footer [1421135896, 1421136117) size=221 gap_from_prev=0 +EOF=1421136117 end_of_last_region=1421136117 gap=0 +``` + +### SHA1 verification (step 5) + +All three hashes verify on `pakchunk0`, and on all 34 paks: + +```text +sha1(primary index) = 5d2f07c9209f8a50f17e15a3f6562e07d85b9ab0 == footer IndexHash MATCH +sha1(path-hash idx) = 52ce88320eae17b776eea8d6e1344d28a19728ac == primary PathHashIndexHash MATCH +sha1(full-dir idx) = 9a2878db6f62cf566f591c681fa58ba55973ca1e == primary FullDirectoryIndexHash MATCH +``` + +## Encoded `FPakEntry` bit-packed format + +Each record starts with a `uint32` flags word: + +| bits | meaning | +| ----- | ------------------------------------------------------------------------------------------- | +| 31 | `Offset` is 32-bit (else 64-bit) | +| 30 | `UncompressedSize` is 32-bit (else 64-bit) | +| 29 | `Size` is 32-bit (else 64-bit) | +| 28–23 | `CompressionMethodIndex` (6 bits; index into the footer's `CompressionMethods`, 0 = stored) | +| 22 | encrypted | +| 21–6 | compression block count (16 bits) | +| 5–0 | `CompressionBlockSize >> 11`, or `0x3f` = escape (explicit `uint32` follows) | + +Then, in this exact order: + +```text +if (flags & 0x3f) == 0x3f : uint32 CompressionBlockSize // NOTE: precedes Offset +Offset : uint32 if bit31 else uint64 +UncompressedSize : uint32 if bit30 else uint64 +Size : uint32 if bit29 else uint64 // ONLY when CompressionMethodIndex != 0 + // (when 0, Size == UncompressedSize, not serialized) +if blockCount > 0 && (blockCount > 1 || encrypted): + blockCount x uint32 // per-block compressed length +``` + +The `CompressionBlockSize`-before-`Offset` ordering is the detail that broke the first +decode attempt; it was recovered by using the directory index's entry locations as ground +truth for record boundaries, then fitting fields to the observed record widths. + +### Verification (step 2) + +Sequential decode of `pakchunk0`'s blob consumes **163,940 / 163,940 bytes exactly** and +yields **exactly 9295 records == `NumEntries`**: + +```text +compression-method-index histogram: {0: 4089, 1: 4138, 2: 1068} (0=stored, 1=Oodle, 2=Zlib) +encrypted entries: 0 +32-bit-safe bits: off32=9295 usz32=9295 sz32=9295 (all entries use the 32-bit forms) +entries with offset+size outside the data region [0,1420424182): 0 +multi-block entries whose per-block sizes don't sum to Size: 0 +entries carrying an explicit block-size list: 839 +record-length histogram: {12: 4089, 16: 18, 20: 4349, 28: 615, 32: 78, 36: 40, 40: 23, + 44: 28, 48: 12, 52: 5, 56: 5, 60: 3, 64: 9, 76: 2, 88: 2, + 96: 2, 108: 9, 128: 2, 152: 3, 220: 1} +``` + +**Not all entries are uncompressed** — the task brief's expectation is falsified. Across +all 34 paks: `{0: 44744, 1: 127266, 2: 1068}` — i.e. **74% Oodle**, 25% stored, <1% Zlib. + +**The stored (`CompressionMethodIndex == 0`) shape is exactly 12 bytes** — flags word +`0xE0000000`, `uint32 Offset`, `uint32 UncompressedSize` — for all 4089 such entries in +`pakchunk0`. That is precisely the record the Task 4 writer must emit: + +```text +000000e0 00000000 ab020000 -> flags=0xE0000000, Offset=0, Size=UncompressedSize=683 +``` + +Sanity-check of the compressed shape (block sizes sum to `Size`, verified on every +multi-block entry in all 34 paks): + +```text +flags=0xe08000ff -> cmi=1, blocks=3, blkraw=0x3f + CompressionBlockSize=1048576 Offset=1048576 UncompressedSize=2793472 Size=2584907 + blocks=[996695, 990845, 597367] sum=2584907 == Size MATCH +``` + +## Full directory index + +```text +int32 DirCount +repeat DirCount: + FString DirName // trailing '/', NO leading '/', except the root dir which is exactly "/" + int32 FileCount + repeat FileCount: + FString FileName // leaf name only + int32 PakEntryLocation +``` + +`PakEntryLocation >= 0` is a byte offset into `EncodedPakEntries`. (Negative values would +index the non-encoded `Files` array; **zero negatives observed across all 173,078 entries**, +so the reader should treat them as a hard unsupported-format error.) + +### Verification (step 3) + +```text +pakchunk0: directories=631 path->location mappings=9295 == NumEntries MATCH + consumed 345973/345973 bytes exactly + every location resolves to a decoded record start: True negatives: 0 +sample dirs: ['Engine/Content/', 'Engine/', '/', 'Engine/Content/EngineResources/'] +``` + +Three sample paths (`pakchunk0` carries no `.json`; those live in `data.pak`): + +```text +'Engine/Config/Base.ini' -> loc 105252 +'Engine/Config/BaseCompat.ini' -> loc 105264 +'Engine/Config/BaseDeviceProfiles.ini' -> loc 105284 +``` + +…and from the pak that does have JSON: + +```text +data.pak: 'Factions/D_Factions.json', 'Quests/Modifiers/D_QuestWeatherModifiers.json', + 'Development/D_LevelSequences.json' (298 entries, all .json) +``` + +The full mount-relative path is `DirName + FileName`, which yields a **leading `/` only for +root-directory files** (`"/" + "DataTableMetadata.json"`). Strip that leading `/` to get the +canonical mount-relative path. + +## Path hash index + +The path-hash _region_ holds two structures back to back: + +```text +int32 Count +repeat Count: uint64 PathHash ; int32 PakEntryLocation + // same wire format as the full directory index +``` + +### Verification (step 4) + +```text +pakchunk0: hash entries=9295 == NumEntries MATCH + map ends at byte 111544 of 201687 + trailing 90143 bytes parse as a second (pruned) directory index: + dirs=401 files=3696, consuming 201687/201687 exactly + pruned entries are a strict subset of the full index: True +``` + +**33 of the 34 paks ship an EMPTY pruned directory index** (`DirCount == 0`, i.e. a bare +`int32 0`); only `pakchunk0` populates it. Emitting an empty pruned index is therefore +demonstrably a shape the engine loads. + +### The hash recipe — determined by making the numbers match + +```text +h := 0xCBF29CE484222325 + PathHashSeed (uint64 wrapping ADD, not XOR) +for each byte b of UTF16LE(lowercase(mount-relative path, leading '/' stripped)): + h ^= b + h *= 0x00000100000001B3 (uint64 wrapping multiply) +``` + +That is standard **FNV-1a 64** with the offset basis _added_ to the seed, hashing the +UTF-16LE bytes of the lowercased path with **no NUL terminator**. A brute-force sweep over +{basis, seed, basis^seed, basis+seed} × {UTF-16LE, UTF-16LE+NUL, UTF-8, UTF-8+NUL} × +{lower, upper, as-is} × {as-is, strip-leading-`/`, add-leading-`/`, backslashes} left +`basis+seed / UTF-16LE / lower / strip-leading-'/'` as the only recipe that matches. +`basis+seed` and `basis^seed` are distinguishable here (the seeds are ≤ 32 bits and the +add carries into bit 32), and only the ADD form matches. + +Verified not on 3 paths but on **every path in every pak** — each computed hash is present +in the map _and_ maps to the same `PakEntryLocation` the directory index gives: + +```text +pakchunk0 9295/9295 data.pak 298/298 ... all 34 paks: 173,078/173,078 MATCH +``` + +Worked examples (`pakchunk0`, seed `0x9c4dd25a`): + +```text +'Engine/Config/Base.ini' -> 0xaef1d2bae819faf6 -> loc 105252 +'Engine/Config/BaseCompat.ini' -> 0x53ea9a367ce73eda -> loc 105264 +'Engine/Config/BaseDeviceProfiles.ini' -> 0x6d1b238f9ca74f86 -> loc 105284 +``` + +**The leading-`/` strip is load-bearing.** An initial sweep that concatenated +`DirName + FileName` verbatim passed on 29 of 33 chunks but failed on 5 (e.g. +`pakchunk0_s9`: only 111 of 3624 paths matched) — precisely the chunks with many +root-directory files, whose paths came out as `/M_DEP_Crate_Sinotai_D.uasset`. Stripping +the leading `/` took every pak to 100%. + +Note: no non-ASCII path exists in any of the 34 paks, so ASCII-only vs. full-Unicode case +folding is not distinguishable here. Our writer controls its own paths, so lowercasing +ASCII `A-Z` matches UE's `FChar::ToLower` for everything we will emit. + +## Per-entry local header (step 6) + +Each file's data region begins with a **full, non-encoded `FPakEntry`** re-serialized +inline, immediately followed by the payload: + +```text +int64 Offset // ALWAYS 0 in the local copy — not the absolute offset +int64 Size // on-disk (post-compression) size +int64 UncompressedSize +int32 CompressionMethodIndex +[20] Hash // SHA1 of the ON-DISK payload bytes +if CompressionMethodIndex != 0: + int32 BlockCount + repeat BlockCount: int64 CompressedStart ; int64 CompressedEnd // relative to entry start +uint8 Flags // 0 = not encrypted, not deleted +uint32 CompressionBlockSize +``` + +**Stored entries: exactly 53 bytes** (8+8+8+4+20+1+4). The plan's assumed 49 is wrong — it +omits the trailing `uint32 CompressionBlockSize`. Compressed entries: `53 + 4 + 16*BlockCount`. + +Note `Flags` and `CompressionBlockSize` come **after** the block list, not before it. + +Cross-check on a real JSON entry from `data.pak`, reached through the full production read +path (path → FNV-1a hash → path-hash index → encoded record → local header → payload): + +```text +path 'Factions/D_Factions.json' +encoded entry: flags=0xe0000000 offset=816821 size=113 usize=113 cmi=0 rec_len=12 +local header (53 B): Offset=0 Size=113 UncompressedSize=113 CMI=0 Flags=0x00 CompressionBlockSize=0 +local Hash = 6b899b02ef58ae54ac64a2f7acf929530c995b29 +sha1(payload) = 6b899b02ef58ae54ac64a2f7acf929530c995b29 MATCH +local Size/UncompressedSize agree with the index entry: True +local Offset field is ZERO (not the absolute offset): True +payload parses as JSON: True +payload head: {\r\n "RowStruct": "/Script/Icarus.Factions",\r\n "Defaults": {},\r\n ... +``` + +Raw 53-byte header of a stored entry, for byte-level reference: + +```text +00000000 00000000 Offset = 0 +10000000 00000000 Size = 16 +10000000 00000000 UncompressedSize = 16 +00000000 CompressionMethodIndex = 0 +a897c5aa2519d4fb9b31c4555aa3a62b297d9e55 Hash (SHA1 of payload) +00 Flags = 0 +00000000 CompressionBlockSize = 0 +``` + +`Hash` covers the **on-disk** bytes. Proven by fully decompressing a Zlib entry +(`CompressionMethodIndex 2`) — the only compressed codec the stdlib can read: + +```text +path 'Engine/Plugins/Runtime/HairStrands/Config/BaseHairStrands.ini' +cmi=2 blocks=1 size=286 usize=1424 blocksize=1424 local header = 73 bytes (53+4+16) +local blocks (start,end) = [(73, 359)] <- first block starts exactly at the header end +zlib-decompressed 1424 bytes == UncompressedSize MATCH +sha1(compressed payload) == local Hash MATCH +decompressed head: b'[Startup]\r\nfx.UseShaderStages=1\r\n\r\n[CoreRedirects]\r\n...' +``` + +## Data-region packing and alignment + +Entries are packed **contiguously** — `offset(n+1) == offset(n) + headerSize(n) + size(n)` — +for 8385 of `pakchunk0`'s 9294 adjacent pairs, and the last entry ends at exactly +`IndexOffset` (1420424182), so the data region is flush against the index. + +The remaining **909 pairs have a padding gap**: every post-gap entry begins at a **1 MiB-aligned** +offset (the cooker's compression-block alignment; 413,219,809 bytes of padding total). A +reader must therefore always seek to each entry's recorded `Offset` and never assume +contiguity. Our writer packs contiguously with no alignment, which is valid — 8385 real +adjacent pairs demonstrate zero padding is accepted. + +## Cross-check across every pak on disk (step 7) + +Rather than spot-checking `pakchunk0_s10`, the full decode was run against **all 34 paks**. +Every one passes every invariant: version 11, 221-byte footer, `bEncryptedIndex == 0`, all +three SHA1 gates, encoded blob consumed exactly, `NumNonEncodedFiles == 0`, zero trailing +bytes in the primary index, decoded-record count == full-dir-index count == path-hash count +== `NumEntries`, zero out-of-range offsets, all block sizes summing to `Size`, and 100% +path-hash agreement. + +```text +chunk ver entries dec fdi phi sha1x3 hash blob oob blk pruned +pakchunk0-WindowsNoEditor.pak 11 9295 9295 9295 9295 True 9295 True 0 0 3696 OK +pakchunk0_s10-WindowsNoEditor.pak 11 3078 3078 3078 3078 True 3078 True 0 0 0 OK +pakchunk0_s20-WindowsNoEditor.pak 11 42361 42361 42361 42361 True 42361 True 0 0 0 OK +... (all 33 chunks) ... +data.pak 11 298 298 298 298 True 298 True 0 0 0 OK + +ALL PAKS PASS: True +``` + +`pakchunk0_s10` specifically: `mount='../../../Icarus/Content/ASS/'`, 3078 entries, +seed `0x659d16d2`, empty pruned index, and a verified 53-byte local header + +payload-SHA1 match on `DPS/SK_DPS_SML_DropShip_02_TOP_Skeleton.uexp` (558 bytes). + +## Part 2 verdict + +**CONFIRMED.** The version-11 index format is fully decoded and every structure the Task 2 +reader and Task 4 writer need is specified above at byte level, verified against 173,078 +real entries in 34 paks. Two findings change the plan's scope materially: + +1. The classic flat index the plan assumed **does not exist** in version 11 — the reader + needs the primary index + encoded-entry decoder + full directory index, and the writer + must emit all three index structures plus the path-hash index (recipe above). +2. **Oodle.** The real `.EXMOD` base pak is `Content/Data/data.pak`, and 258 of its 298 + JSON tables — including every table worth patching — are Oodle-compressed. The + uncompressed-only **read** path cannot reach them. Unresolved; needs a product decision. + → **Resolved in Part 3** by a user decision: base tables come from hosted community + dumps instead of local Oodle decompression. + + > **CORRECTED 2026-08-01 (#175):** `data.pak` is Zlib, not Oodle — see the correction banner in Part 3. + +--- + +# Part 3 — Hosted base-table dumps (#136 plan rev3 spike) + +Spike 3 grounds the user decision that resolves Part 2's Oodle blocker: the compile +pipeline takes base data tables from the community's hosted per-week JSON dumps rather +than decompressing the local `data.pak`. Everything below was fetched over the network and +byte-checked against the local install on 2026-07-31. + +## ⚠ Two findings that qualify the decision + +Read these before relying on this strategy. + +1. **IMM does not do this.** The decision was framed as "IMM's approach", but Icarus Mod + Manager's own README describes local extraction: _"it should unpak the data folder from + the game"_, via an "Update data folder" button, and _"you will need to add the oodle + compression plugin to your unrealPak folder"_. IMM solves Oodle by shipping the Oodle + plugin alongside UnrealPak, not by downloading dumps. Hosted dumps are a real and + separate thing (documented below), but they are not IMM's mechanism. + Source: +2. **The freshest dump is 7 weeks behind this install.** The installed game is **Week 243**; + the best-maintained dump repo's HEAD is **Week 236** (2026-06-12). So _right now_ there + is no dump matching this machine's game build, and the pipeline's fail-loud path would + trigger. The maintainer has also gone dormant before (no commits Dec 2024 → Jul 2025). + This is a live availability risk, not a theoretical one. + +## 1. Where the dumps live, what they are, who maintains them + +**Primary source — `GODOFMINECRAFT4/IcarusData`** (GitHub, branch `master`). + +| | | +| ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Repo | | +| README | "Icarus Data.pak Unpack — This Repo Will be Updated Each Update To Show What Files Were Edited" | +| Maintainer | GODOFMINECRAFT4 (single personal repo, 0 stars, created 2025-07-18) | +| Format | The unpacked `data.pak` tree as **loose JSON files committed at repo root** — `AI/`, `Accolades/`, `Items/`, … (282 root-level `.json`). Not a zip, not a release asset. | +| Versioning | **Git commits only.** One commit per game week; the week appears _only in the commit message_ (`"Week 236 data.pack Unpacked Using New Semi Automated Workflow"`). **No tags, no releases, one branch.** | +| Tooling | QuickBMS (`quickbms.exe`, `unreal_pak.bms`, `reimport*.bat` are committed alongside). QuickBMS has Oodle support including on Linux. | +| Coverage | Weeks 149 → 236 in history, with a dormancy gap (Week 160 Dec 2024 → Week 189 Jul 2025, commit message _"IM BACK BITCHES"_). | + +A stale `data/` subdirectory (284 JSON) also sits in the repo; the authoritative current +tree is the **root-level** one. Do not read `data/`. + +### Verified URL patterns + +All fetched successfully, no authentication, no rate-limit issues: + +```text +# Whole tree at HEAD (tar.gz) — 36,391,684 bytes, ~3.7 s +https://codeload.github.com/GODOFMINECRAFT4/IcarusData/tar.gz/refs/heads/master + +# Whole tree at a specific week (by commit SHA) — verified on Week 231 (ef2b5e11) +https://codeload.github.com/GODOFMINECRAFT4/IcarusData/tar.gz/ # 36,372,614 bytes +https://github.com/GODOFMINECRAFT4/IcarusData/archive/.zip # HTTP 200 + +# One table (raw blob) — D_ItemsStatic.json = 7,040,520 bytes in 0.285 s +https://raw.githubusercontent.com/GODOFMINECRAFT4/IcarusData//Items/D_ItemsStatic.json + +# Week index (commit list; week parsed from commit message) +https://api.github.com/repos/GODOFMINECRAFT4/IcarusData/commits?per_page=100 +``` + +### Secondary sources (all staler — fallbacks or cross-checks only) + +| Source | State | +| ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `MatthiasKunnen/icarus-pedia` → `gamedata/data_pak/` | Week 218 (2026-02-09). Valuable for one reason: its commit messages carry an **explicit week↔version mapping**, e.g. `"Upgrade to week 218 REV. 2.3.29.148374-SHIPPING-GREATHUNTS"`. | +| `Jimk72/Icarus_Software` → `data.zip` | 265 tables, 24.6 MB uncompressed. Uploaded **once**, 2024-04-26, never updated. Not per-week. | +| `NateEkat/Icarus-DataExport` | "The data.pak from the steam game Icarus", last pushed 2025-09-22. | +| `Jimk72/Icarus_Software` → `DUMP_Week_140.zip` | **Not data tables** — a C++ SDK header dump (3,477 `*_classes.h`/`*_struct.h`). Irrelevant here despite the promising name. | + +## 2. Week scheme and local build detection + +### The dump side + +Week numbers exist only in commit messages. Parsing `Week (\d+)` from +`/repos/GODOFMINECRAFT4/IcarusData/commits` yields the week→SHA index; the SHA then +addresses that week's tree. Mod `compatibility` strings in the Firestore catalog use a +different, shorter form (e.g. `"w57"`), so any mapping between mod compatibility and dump +week is a string-normalization problem, not a lookup. + +### The local side — there is NO week number in the install + +Searched the whole install: `Icarus/Config/` holds only `SettingsSchema.json`, +`TestRails.json` and `version.json`, and none names a week. The two authoritative local +facts are: + +```jsonc +// /Icarus/Config/version.json — the canonical build identity +{ + "Name": "Icarus", + "Version": { + "Major": 3, + "Minor": 0, + "Patch": 21, + "Changelist": 155335, + "BuildType": "Shipping", + "FeatureLevel": "DangerousHorizons", + }, + "Data": { "Changelist": 155151 }, // <- the data.pak's own changelist +} +``` + +```text +# /steamapps/appmanifest_1149460.acf +"buildid" "24487768" +"LastUpdated" "1785533918" -> 2026-07-31 21:38:38 UTC +``` + +### Bridging build → week + +The Steam News API supplies the missing link, unauthenticated: + +```text +https://api.steampowered.com/ISteamNews/GetNewsForApp/v2/?appid=1149460&count=12 +``` + +```text +2026-07-31 Hotfix Version 3.0.21.155335-rel-DangerousHorizons <- matches version.json exactly +2026-07-30 Icarus Week 243 Update | Livewire Revamp +2026-07-24 Icarus Week 242 Update | Workshop Flashlight +... +2026-06-12 Icarus Week 236 Update | Ubis Husbandry & Phenotypes <- dump repo HEAD +``` + +`Major.Minor.Patch.Changelist` from `version.json` reproduces the hotfix title verbatim +(`3.0.21.155335`), and the nearest preceding "Icarus Week N Update" gives the week. +**Installed build = Week 243.** + +**Recommended recipe — do not depend on news parsing.** Steam titles are prose and will +drift. The robust detection is _content-based_, and it is available offline: + +> Read `version.json` for identity/reporting, then **prove** a candidate dump matches the +> install by byte-comparing the tables that `data.pak` stores **uncompressed**. Those 40 +> tables are readable with `internal/unrealpak` alone — no Oodle — so the check costs +> nothing and is exact. If they all match, the dump is the right week; if any differs, it +> is not. + +That check is what detected the 7-week drift below. + +## 3. Cross-validation: dump vs. local `data.pak` + +Compared the dump at HEAD (Week 236) against the installed Week 243 `data.pak`, reading +stored entries through the Part 2 reader logic. + +### Normalization: line endings, and nothing else + +The shipped pak stores **CRLF**; the git blobs are **LF** (committed with autocrlf; the +repo has no `.gitattributes`, and `raw.githubusercontent.com` also serves LF). The +transform is exact and reversible: + +```text +AI/D_AIEvents.json: pak = 910 bytes (26 CRLF) | dump = 884 bytes (26 LF) +pak == dump.replace(b"\n", b"\r\n") -> True # byte-for-byte +pak.replace(b"\r\n", b"\n") == dump -> True +``` + +So a fetcher must restore `LF -> CRLF` to reproduce shipped bytes. No other normalization +exists — no re-indentation, no key reordering, no encoding change. + +### Results (after LF→CRLF restoration) + +| Check | Result | +| --------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | +| Stored (uncompressed) tables, byte-compare | **37 / 40 byte-identical** | +| Stored tables genuinely differing | 3 — `Audio/D_MusicTrackStateGroups.json`, `Audio/MusicConditions/D_MusicLocationConditions.json`, `Audio/MusicConditions/D_MusicQuestConditions.json` | +| Oodle tables, dump size vs index `UncompressedSize` | 167 / 252 exact; 85 differ (all smaller in the dump) | +| Tables in `data.pak` with no dump counterpart | 6 — `Settlement/D_SettlementNPCClothing/Items/Skills/Traits`, `Settlement/D_SettlementRaids`, `Tools/D_NPCWeapon` (added after Week 236) | + +**Verdict: the dumps are faithful, and this dump is the wrong week.** The 37 exact matches +prove fidelity — an unpack that round-trips 37 files byte-perfectly is not lossy. The 3 +content differences, 85 size differences and 6 missing tables are real game changes between +Week 236 and Week 243, exactly what a 7-week gap predicts. + +Note `data.pak` contains **no Zlib entries** (only 40 stored + 258 Oodle), so the planned +"compare 1 Zlib table" check was not applicable here; the 252-table `UncompressedSize` +comparison substitutes for it and covers far more ground. (Zlib entries do exist in +`pakchunk0`, and Part 2 already proved stdlib zlib round-trips them.) + +> **CORRECTED 2026-08-01 (#175):** `data.pak` is Zlib, not Oodle — see the correction banner in Part 3. + +For reference, an earlier comparison against `Jimk72/Icarus_Software/data.zip` gave +29/37 byte-identical with no line-ending difference, plus 34 missing tables — consistent +with that snapshot being an older week from a different unpack toolchain. + +## 4. Availability and robustness facts (for error handling) + +| Fact | Measured | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | +| Authentication | **None.** Plain public HTTPS GETs. | +| Full-tree download | 36.4 MB tar.gz, **~3.7 s** | +| Single-table download | `D_ItemsStatic.json` 7,040,520 B in **0.285 s** | +| Repo size | ~40 MB | +| Historical weeks | **Retrievable by commit SHA** — verified fetching Week 231 (`ef2b5e11`). Git history is permanent; there are no releases or tags to expire. | +| Rate limits | GitHub anonymous API limits apply to the commits listing (60/hr/IP); `raw`/`codeload` downloads are not API-limited. Cache the week index. | +| Freshness risk | **HIGH.** HEAD is Week 236 vs installed Week 243. Prior dormancy: Dec 2024 → Jul 2025. Single maintainer, 0 stars, no CI. | +| Single point of failure | One personal repo. `icarus-pedia` (Week 218) is the only comparable fallback and is even staler. | + +## Local dump-directory override (rev4) + +Because the hosted dump can lag the installed game — it was 7 weeks behind at spike time — +the pipeline also accepts a user-supplied directory holding an unpacked `data.pak` JSON +tree (QuickBMS output, IMM's extracted `data` folder, or anything with the same layout), +configured per game as `data_dump_path` in `games.yaml`. When set it replaces the network +fetch entirely, which additionally makes offline compiles possible. Crucially it does **not** +relax the correctness gate: the same byte-comparison against the 40 tables `data.pak` stores +uncompressed runs on local directories exactly as it does on fetched ones, so a local +extraction from the wrong week is rejected with the same error, additionally naming the +configured path. Verified on real data both ways — the Week 236 tree pointed at as a local +directory against this Week 243 install is refused identically to the hosted fetch, while a +directory built from the install's own 40 stored tables validates clean. One wrinkle worth +recording: a local extraction may already hold CRLF (QuickBMS writes what the pak stored) +whereas git blobs are LF, so the CRLF restoration is written to be idempotent and both +sources converge on the shipped byte shape. + +## Part 3 verdict + +**VERIFIED, WITH A MATERIAL CAVEAT.** Per-week hosted dumps exist, are addressable by +commit SHA, need no authentication, download in seconds, and are **byte-faithful modulo a +deterministic LF→CRLF transform** (37/40 stored tables byte-identical). The Oodle blocker +from Part 2 is genuinely resolved by this route. + +The caveat is availability, not fidelity: the freshest dump is Week 236 while this install +is Week 243, so the pipeline must treat "no dump for the installed build" as a normal, +well-handled, fail-loud outcome rather than an edge case — it is the current state on this +machine. And the "IMM does this" premise is incorrect; IMM extracts locally with an Oodle +plugin. diff --git a/docs/plans/archive/icarus-quickbms-spike-findings.md b/docs/plans/archive/icarus-quickbms-spike-findings.md new file mode 100644 index 0000000..1633885 --- /dev/null +++ b/docs/plans/archive/icarus-quickbms-spike-findings.md @@ -0,0 +1,327 @@ +# Icarus QuickBMS Auto-Extraction — Task 1 Spike Findings (#174) + +Empirical, read-only spike run 2026-08-01 on the reference machine. No product code was +written. No `sudo`, no system packages installed, no game files modified. + +## Verdict, up front + +**The design's premise is FALSIFIED — twice over, and the second falsification is good news.** + +1. **Linux QuickBMS cannot decompress Icarus's Oodle data.** QuickBMS 0.12.0's Oodle support + is the open-source `powzix/kraken` reimplementation, and on real Icarus Oodle blocks it + **crashes the process** (SIGSEGV / assertion abort) on **19 of 20** sampled blocks. That is + the literal stop-gate this task was written to test, and it fails. + +2. **It does not matter, because `Content/Data/data.pak` contains no Oodle at all.** Its + footer declares `CompressionMethods = ["Zlib"]` — so its 258 compressed tables are + **Zlib**, not Oodle. All 258 decompress with stdlib zlib into valid JSON. The Oodle + blocker that motivated both the hosted-dump strategy (#136 r3/r4) and this entire feature + **never applied to the pak the compile pipeline actually reads.** + +**Recommendation: do not implement #174 as designed.** Replace it with ~15 lines of stdlib +`compress/zlib` support in `internal/unrealpak`, which makes the installed `data.pak` fully +readable, week-correct by construction, and removes the need for hosted dumps, QuickBMS, +`auto_extract`, `quickbms_path`, and the four-leg source chain. See "What to do instead". + +--- + +## 1. Package availability and recommended permanent install route + +```text +AUR: quickbms 0.12.0-2 votes=10 last modified 2024-01-11 out-of-date=no + "Files extractor and reimporter, archives and file formats parser..." + URL: http://aluigi.altervista.org/quickbms.htm +command -v quickbms -> not on PATH (nothing was previously installed) +``` + +**Recommended permanent route (if QuickBMS is ever wanted for other reasons): the AUR package.** +Reasoning: upstream's 0.12.0 source (dated Aug 2022) **does not compile** on this machine's +toolchain (GCC 16.1.1) without patching. These are structural C23/OpenSSL-3 breakages, not +warning noise, spread across the tool's own sources and its vendored libraries: + +```text +crc.c:126 error: too many arguments to function 'add_func'; expected 0, have 5 +included/compresslayla.c error: too many arguments to function 'CompressLAYLA_func'; expected 0, have 7 +included/microvision.c error: too many arguments to function 'microvision_decompress_func'; expected 0, have 9 +compression/camoto.c:34 error: 'bool' cannot be defined via 'typedef' +compression/de_compress.c error: too many arguments to function 'core_bytes'; expected 0, have 2 +perform.c:1544 error: 'RSA_SSLV23_PADDING' undeclared (removed in OpenSSL 3) +libs/TurboRLE, libs/libkirk: implicit declarations of abort/memcpy (hard errors since GCC 14) +``` + +C23 made `f()` mean "takes no arguments", which invalidates this codebase's K&R-style +declarations; `bool` became a keyword; and OpenSSL 3 removed `RSA_SSLV23_PADDING`. Passing +`-Wno-implicit-function-declaration` clears only the warning-class errors, and the C++ sources +reject those flags, so the build still fails. Patching this is a packager's job — which is +exactly the value of the AUR route. The no-root fallback is upstream's **prebuilt Linux +binary** (see §2), which works as-is. + +This is recorded for completeness only — nothing in the recommended plan (below) needs +QuickBMS. + +## 2. What was built/obtained, and the version banner + +Upstream's `papers/quickbms.zip` is the **Windows binary distribution** (`quickbms.exe`, +`quickbms_4gb_files.exe`, docs) — it contains no source and no Makefile, so the brief's +`make`-in-that-tarball step could not apply. The two correct artifacts are: + +```bash +# Prebuilt Linux binaries (what this spike used): +curl -sL -o quickbms_linux.zip https://aluigi.altervista.org/papers/quickbms_linux.zip +unzip -o -q quickbms_linux.zip && chmod +x quickbms quickbms_4gb_files +# Full source (does not build on GCC 16 unpatched): +curl -sL -o quickbms-src-0.12.0.zip https://aluigi.altervista.org/papers/quickbms-src-0.12.0.zip +``` + +```text +binary: ~/.local/src/quickbms/linux/quickbms +file: ELF 32-bit LSB executable, Intel i386, statically linked, for GNU/Linux 2.6.24, stripped + (runs fine on x86_64 — the kernel has ia32 emulation and the binary is static) + +banner: QuickBMS generic files extractor and reimporter 0.12.0 + by Luigi Auriemma + (Aug 24 2022 - 10:26:51) +``` + +The binary was left at `~/.local/src/quickbms/linux/quickbms` and deliberately **not** copied +onto `PATH`, since the recommendation is to not use it. + +## 3. The `.bms` script and its license + +**The ecosystem script the brief pointed at does not exist.** Both "scripts" committed to +`GODOFMINECRAFT4/IcarusData` are failed downloads the maintainer committed by accident: + +| Repo file | Size | Actual content | +| ----------------------- | ------- | --------------------------------- | +| `unreal_pak.bms` | 14 B | the literal text `404: Not Found` | +| `unreal_pak_script.bms` | 2,579 B | aluigi's 404 HTML error page | +| `quickbms_scripts.zip` | 9 B | stub | + +The real canonical script is **`unreal_tournament_4.bms`**: + +```text +URL: https://aluigi.altervista.org/bms/unreal_tournament_4.bms +size: 10,336 bytes (347 lines) +sha256: fbe5c57c9b787a7b84d808a12042764e0a35615c056c5c72b40af4786a439344 +header: "# Unreal Engine 4 - Unreal Tournament 4 (*WindowsNoEditor.pak) (script 0.4.25)" + "# script for QuickBMS http://quickbms.aluigi.org" +``` + +### License verdict: **MURKY** + +Searched the script for `licen|copyright|gpl|public domain|free|redistribut|permission`: +**no matches — the script carries no license statement of any kind.** The QuickBMS _manual_ +states the **tool** is GPL-2.0 ("The tool is open source under the GPL 2.0 license... You can +distribute the original quickbms.exe file as you desire but reusing its source code and/or +modifying it may require the same or compatible open source license"), and the source tarball +ships `gpl-2.0.txt` — but that grant is scoped to the tool's own source, not to the `.bms` +script files published on the website. The QuickBMS site page carries no redistribution grant +for the scripts either. + +Per the brief's own rubric, "no grant, **or** terms that restrict redistribution" → **MURKY**, +which would have flipped Tasks 2–3 from `go:embed` to download-on-demand + cache. Moot under +the recommendation below, but recorded so the decision is not re-litigated. + +### The script also does not support this pak + +Line 108 of `unreal_tournament_4.bms` reads: + +```text +# mobile version 10 is not supported +``` + +and the index parser that follows assumes the **classic flat index** (`get FILES long` then +per-entry records). Icarus's paks are **version 11**, which uses the three-part index +(primary + path-hash + full-directory) with _bit-packed encoded_ entries — the layout pinned +byte-for-byte in `icarus-pak-format-findings.md` Part 2. So the script cannot enumerate this +pak at all, independent of any compression question. + +## 4. Extraction attempt: invocation, exit code, output, runtime + +```bash +~/.local/src/quickbms/linux/quickbms -o \ + ~/.local/src/quickbms/unreal_tournament_4.bms \ + /data/SteamLibrary/steamapps/common/Icarus/Icarus/Content/Data/data.pak \ + ~/.local/src/quickbms/extracted +``` + +```text +exit=3 real 0m0.027s + + offset filesize filename +-------------------------------------- +Error: incomplete input file 0: .../Content/Data/data.pak + Can't read 6 bytes from offset 00258477. + coverage file 0 0% 22382 2458743 . offset 00258477 +Last script line before the error or that produced the error: + 159 get CHUNK_OFFSET longlong TOC_FILE +``` + +Zero files extracted. This is the version-11 index incompatibility from §3, not a compression +failure — the script died while parsing the index. + +## 5. Output layout + +**Not determinable: no extraction ever succeeded.** `$OUT` remained empty (0 `.json` files), +so there is no `DataTableMetadata.json` location and no mount-path-prefix question to answer. +Any future extractor work must re-establish this. + +## 6. Gate results + +The brief's gate program compares QuickBMS output against `unrealpak` ground truth. With no +extraction output, that comparison is vacuous. The two substantive questions were answered +directly instead. + +### 6a. Can QuickBMS decompress Icarus's Oodle on Linux? **No.** + +Probed with a minimal script against real single-block Oodle entries from +`pakchunk0-WindowsNoEditor.pak` (offsets/sizes derived from the verified v11 index decoder): + +```bms +comtype oodle +clog "out.bin" +``` + +```text +20 real Icarus Oodle blocks: success=1 fail/crash=19 +``` + +Failures are **hard crashes, not clean errors**: + +```text +quickbms: libs/powzix/kraken.cpp:235: void BitReader_RefillBackwards(BitReader*): + Assertion `bits->bitpos <= 24' failed. -> exit 134 (SIGABRT, core dumped) +Segmentation fault (core dumped) -> exit 139 +``` + +`quickbms_4gb_files` fails identically (exit 139 on a block the standard binary also fails). +The one success produced exactly its expected 4,444 bytes, so the invocation form is correct — +the decoder itself is simply not compatible with the Oodle version Icarus ships. QuickBMS's +Oodle is `powzix/kraken`, a clean-room reimplementation that lags current `oo2core` releases. + +A tool that segfaults on 95% of real inputs cannot be the basis of an auto-run fallback. + +### 6b. Does `data.pak` need Oodle at all? **No — it has none.** + +Read from `data.pak`'s own footer: + +```text +version=11 indexOffset=2436670 indexSize=8007 +CompressionMethods = ['Zlib', '', '', '', ''] <- slot 0 is Zlib; there is NO Oodle slot +mount='C:/BA/work/92bbbfa44df12262/Temp/Data/' entries=298 +compression-method-index histogram: STORED=40, Zlib=258 +``` + +**This corrects an error in `icarus-pak-format-findings.md` Part 3.** That document recorded +data.pak as "258 Oodle-compressed" because the rev3 sweep resolved method indices against +`pakchunk0`'s method table (`['Oodle','Zlib']`, where CMI 1 = Oodle) and never read +`data.pak`'s own table (`['Zlib']`, where **CMI 1 = Zlib**). The raw histogram `{1: 258, 0: 40}` +was right; the label attached to index 1 was wrong. + +Method tables across every pak in the install: + +```text +data.pak ver 11 298 entries ['Zlib'] -> STORED=40, Zlib=258 +pakchunk0-WindowsNoEditor.pak ver 11 9295 entries ['Oodle','Zlib'] -> STORED=4089, Oodle=4138, Zlib=1068 +pakchunk0_s1 … s32 (32 chunks) ver 11 … ['Oodle'] -> STORED=…, Oodle=… +paks that actually contain Oodle-compressed entries: 33 of 34 — all of them asset chunks +``` + +Oodle is real in Icarus, but exclusively in the `Content/Paks/pakchunk0*` **asset** chunks +(cooked `.uasset`/`.uexp`/`.ubulk`), which contain **zero `.json`** and which the compile +pipeline never reads for base tables. + +### 6c. Full reconstruction of `data.pak` with stdlib-only primitives + +Decoded every entry and decompressed using only what Go's standard library provides +(`compress/zlib` + `crypto/sha1`): + +```text +STORED tables: 40 ok (SHA1-verified against the entry header, and valid JSON), 0 bad +ZLIB tables: 258 ok (decompressed, size-checked against UncompressedSize, valid JSON), 0 bad +TOTAL: 298/298 +total decompressed bytes: 40,908,881 + +Items/D_ItemsStatic.json : 7,304,687 bytes +Talents/D_Talents.json : 2,626,506 bytes +Factions/D_Factions.json : 113 bytes +``` + +Those figures match `icarus-pak-format-findings.md` Part 3's independently-derived numbers +exactly (40,908,881 total; D_ItemsStatic 7,304,687), which cross-validates both the decoder +and the earlier findings' raw measurements. + +Current behavior, confirmed against the shipped reader: + +```text +unrealpak.Open(data.pak).Files() -> 298 entries +ReadFile("Items/D_ItemsStatic.json") -> unsupported pak feature: compressed entry (method 1) +ReadFile("Factions/D_Factions.json") -> 113 bytes, err= +``` + +So the single blocking line is `ReadFile`'s refusal of `method != 0`. + +## 7. What to do instead of #174 + +Teach `internal/unrealpak` to decompress Zlib entries, keyed on the footer's +`CompressionMethods` name (already parsed but currently discarded): + +- Resolve `CompressionMethodIndex` → method **name** via the footer table; accept `"Zlib"` + (case-insensitive), keep refusing everything else — `"Oodle"` stays a loud + `ErrUnsupportedFormat`, which is correct and honest. +- For a Zlib entry, read the local header's block list (`BlockCount`, then + `CompressedStart`/`CompressedEnd` pairs relative to the entry offset — layout already + documented in Part 2), `zlib.NewReader` each block, concatenate, and verify the total + against `UncompressedSize`. The existing per-entry SHA1 covers the on-disk (compressed) + bytes, so it still applies unchanged. +- Reuse the existing size caps; nothing else in the reader changes. + +Consequences, all favorable: + +- `data.pak` becomes fully readable from the installed game — **week-correct by construction**, + no network, no external binary, no cache, no validation gate needed against a third party. +- The hosted-dump chain (#136 r3/r4), `data_dump_path`, `validateDump`, `auto_extract`, + `quickbms_path`, and the whole four-leg source chain become unnecessary for base tables. + The stale-dump problem that motivated this feature disappears. +- The "compile requires network access" Global Constraint can be dropped — compiling becomes + fully offline. +- `#174` as scoped (embed a `.bms`, detect/invoke an external binary, announce, cache) should + be closed as obsolete rather than implemented. + +Worth confirming before closing #174: whether any _asset_ (non-JSON) use case ever needs the +Oodle chunks. Nothing in the `.EXMOD`/`.EXMODZ` compile path does — mods ship their own +pre-built assets in the `.EXMODZ` — so this looks like a clean removal. + +--- + +## 8. Answers to `SPIKE-CONFIRM:` markers + +The plan (`2026-08-01-icarus-quickbms-fallback.md`) carries 5 in-code markers. All five are +resolved below. **Read the caveat first: the recommendation is to delete these code paths +rather than fill them in.** The values are recorded so the revision pass is mechanical either +way. + +| # | Marker (plan location) | Resolution | +| --- | --------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | **Binary name / path** — Task 2, `quickbmsBinaryName` | `quickbms` is correct as a PATH name, but **there is no working install route on this machine**: upstream 0.12.0 source does not build on GCC 16 unpatched, and the working artifact is upstream's prebuilt **32-bit** static ELF (`quickbms_linux.zip`), left at `~/.local/src/quickbms/linux/quickbms`. AUR `quickbms 0.12.0-2` is the recommended packaged route. `quickbms_4gb_files` behaves identically on Oodle (also crashes), so the base-binary default was the right choice. **Moot under the recommendation.** | +| 2 | **Script filename + license verdict → embed-vs-download flip** — Task 2, `embeddedScriptName` + the `go:embed` directive; Task 3 Step 3 | Filename is **`unreal_tournament_4.bms`**, NOT `unreal_pak.bms` — the IcarusData repo's copies are committed 404 pages (14 B and 2,579 B). Canonical URL `https://aluigi.altervista.org/bms/unreal_tournament_4.bms`, sha256 `fbe5c57c9b787a7b84d808a12042764e0a35615c056c5c72b40af4786a439344`, 10,336 bytes. **License verdict: MURKY** — the script carries no license statement; QuickBMS's GPL-2.0 covers the tool's source, not the published scripts. Per the rubric this **flips Tasks 2–3 to download-on-demand + cache** with URL and checksum pinned. Additionally the script **does not support version-11 paks** ("mobile version 10 is not supported"), so it could not be used even with a license. **Moot under the recommendation.** | +| 3 | **Observed runtime → timeout recommendation** — Task 3, `quickbmsTimeout` | No successful extraction, so no representative runtime exists. The failing run aborted in **0.027 s**; single-block Oodle probes crashed in well under a second. If any future external-tool invocation is added, the `quickbmsWaitDelay` mechanism found necessary during plan authoring is **still required and now doubly justified** — this tool dies by SIGSEGV/SIGABRT, so a `CommandContext` + `WaitDelay` bound is the difference between a clean error and a hung compile. The 10-minute `quickbmsTimeout` was never exercised. **Moot under the recommendation.** | +| 4 | **Output layout** — Task 3, `rootMarkerTable` / `findTreeRoot` | **Undetermined — no extraction succeeded**, output directory stayed empty. The `DataTableMetadata.json`-sentinel approach remains the right design _if_ an extractor is ever added (it is layout-agnostic and needs no prior knowledge), but it is unverified against real tool output. **Moot under the recommendation.** | +| 5 | **Flag order** — Task 3, `runQuickBMS` argument order | **Confirmed correct**: `quickbms [-o] ` is the accepted form — the tool parsed all three arguments and reached script execution every time (it failed later, inside the script or the decoder). `-o` (overwrite without prompting) is **required** for non-interactive use; without it QuickBMS prompts on existing files and would hang an automated run. The plan's assumed order was right; the missing `-o` was not. **Moot under the recommendation.** | + +### Downstream corrections this spike forces (not `SPIKE-CONFIRM:` markers) + +- `icarus-pak-format-findings.md` **Part 3** must be corrected: `data.pak` is **Zlib**, not + Oodle. Its "⚠ BLOCKING RISK: 258 of data.pak's 298 JSON files are Oodle-compressed" section + and the Part 2 verdict's Oodle conclusion are both wrong on the method name (the counts are + right). +- `2026-07-29-icarus-exmod-pak-compilation.md` Task 12's blocker note and the entire rev3/rev4 + hosted-dump rationale rest on the same mistaken premise and need revisiting. + +## 9. Reproduction + +Probe scripts used are in `/tmp/qbms-spike/` (`probe.py` — v11 index decoder + entry finder; +`zlibtest.py` — decompress all 258; `xcheck.py` — method tables across all 34 paks; +`final.py` — full 298-table reconstruction; `find_oodle.py`/`more_oodle.py` — Oodle block +selection). QuickBMS artifacts are under `~/.local/src/quickbms/`. Nothing was installed +system-wide; no game file was modified. From dc7f47ff62e9efbcfacc46e4c0f15a8a92c33753 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sun, 2 Aug 2026 00:31:18 -0400 Subject: [PATCH 90/96] chore: bump version to 1.28.0 --- CHANGELOG.md | 5 ++++- cmd/lmm/root.go | 2 +- docs/man/man1/lmm-auth-login.1 | 2 +- docs/man/man1/lmm-auth-logout.1 | 2 +- docs/man/man1/lmm-auth-status.1 | 2 +- docs/man/man1/lmm-auth.1 | 2 +- docs/man/man1/lmm-completion-bash.1 | 2 +- docs/man/man1/lmm-completion-fish.1 | 2 +- docs/man/man1/lmm-completion-powershell.1 | 2 +- docs/man/man1/lmm-completion-zsh.1 | 2 +- docs/man/man1/lmm-completion.1 | 2 +- docs/man/man1/lmm-conflicts.1 | 2 +- docs/man/man1/lmm-deploy.1 | 2 +- docs/man/man1/lmm-game-add.1 | 2 +- docs/man/man1/lmm-game-clear-default.1 | 2 +- docs/man/man1/lmm-game-detect.1 | 2 +- docs/man/man1/lmm-game-set-default.1 | 2 +- docs/man/man1/lmm-game-show-default.1 | 2 +- docs/man/man1/lmm-game.1 | 2 +- docs/man/man1/lmm-import.1 | 2 +- docs/man/man1/lmm-install.1 | 2 +- docs/man/man1/lmm-list.1 | 2 +- docs/man/man1/lmm-mod-disable.1 | 2 +- docs/man/man1/lmm-mod-edit.1 | 2 +- docs/man/man1/lmm-mod-enable.1 | 2 +- docs/man/man1/lmm-mod-files.1 | 2 +- docs/man/man1/lmm-mod-lock.1 | 2 +- docs/man/man1/lmm-mod-set-update.1 | 2 +- docs/man/man1/lmm-mod-show.1 | 2 +- docs/man/man1/lmm-mod-unlock.1 | 2 +- docs/man/man1/lmm-mod.1 | 2 +- docs/man/man1/lmm-profile-apply.1 | 2 +- docs/man/man1/lmm-profile-create.1 | 2 +- docs/man/man1/lmm-profile-delete.1 | 2 +- docs/man/man1/lmm-profile-export.1 | 2 +- docs/man/man1/lmm-profile-import.1 | 2 +- docs/man/man1/lmm-profile-list.1 | 2 +- docs/man/man1/lmm-profile-reorder.1 | 2 +- docs/man/man1/lmm-profile-switch.1 | 2 +- docs/man/man1/lmm-profile-sync.1 | 2 +- docs/man/man1/lmm-profile.1 | 2 +- docs/man/man1/lmm-purge.1 | 2 +- docs/man/man1/lmm-search.1 | 2 +- docs/man/man1/lmm-source-list.1 | 2 +- docs/man/man1/lmm-source-validate.1 | 2 +- docs/man/man1/lmm-source.1 | 2 +- docs/man/man1/lmm-status.1 | 2 +- docs/man/man1/lmm-tui.1 | 2 +- docs/man/man1/lmm-uninstall.1 | 2 +- docs/man/man1/lmm-update-rollback.1 | 2 +- docs/man/man1/lmm-update.1 | 2 +- docs/man/man1/lmm-verify.1 | 2 +- docs/man/man1/lmm.1 | 2 +- 53 files changed, 56 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75b0588..71048fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.28.0] - 2026-08-02 + ### Added - CLI output is now colorized by default when stdout is a terminal, extending the existing `colorGreen`/`colorRed`/`colorYellow` accent mechanism (previously only used by `deploy`/`verify`) with a full 4-color palette (green/yellow/red/cyan, plus bold/dim) across `list`, `status`, `search`, `update`, `conflicts`, and `mod show`. Table headers are bold+cyan. `lmm list` tints the whole row identically with or without `-v` (the row-tint decision is a single shared helper keyed on the mod's actual state, not the display flag): green for the common enabled+deployed case, yellow for enabled-but-undeployed, dim for disabled. `search` tints an installed mod's whole row green; `update`'s POLICY column colors per row. `status`/`mod show` color their values, not just the odd count: `lmm status -g `'s active profile and per-profile "(active)" marker are green, mod/profile counts are cyan, Link Method is cyan, Last Deploy is green (or dim when never deployed); `mod show`'s Version fields are cyan and its Update policy is colored per state (green for auto, yellow for pinned); `conflicts`' stale winner suffix is yellow; and the existing `✓`/`✗` success/failure markers extend to `update` and `mod`'s confirmation lines. Detection is TTY-aware (piped/redirected output stays plain) and layers on top of the existing `--no-color` flag and `NO_COLOR` env var (presence-only per no-color.org), which continue to work unchanged; `--json` output is never colored. Table color is applied only to already-tabwriter-padded text (accented headers, whole-row tints, or a table's genuinely last column) — never to interior cell values before they reach `text/tabwriter`, which pads columns by raw byte length and would misalign them (#112, #193) @@ -1180,7 +1182,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Comprehensive test coverage for core components - MIT License -[Unreleased]: https://github.com/DonovanMods/linux-mod-manager/compare/v1.27.1...HEAD +[Unreleased]: https://github.com/DonovanMods/linux-mod-manager/compare/v1.28.0...HEAD +[1.28.0]: https://github.com/DonovanMods/linux-mod-manager/compare/v1.27.1...v1.28.0 [1.27.1]: https://github.com/DonovanMods/linux-mod-manager/compare/v1.27.0...v1.27.1 [1.27.0]: https://github.com/DonovanMods/linux-mod-manager/compare/v1.26.0...v1.27.0 [1.26.0]: https://github.com/DonovanMods/linux-mod-manager/compare/v1.25.0...v1.26.0 diff --git a/cmd/lmm/root.go b/cmd/lmm/root.go index eaae6e3..92de74f 100644 --- a/cmd/lmm/root.go +++ b/cmd/lmm/root.go @@ -39,7 +39,7 @@ var ErrCancelled = errors.New("cancelled") var ErrReported = errors.New("already reported") var ( - version = "1.27.1" + version = "1.28.0" // Global flags configDir string diff --git a/docs/man/man1/lmm-auth-login.1 b/docs/man/man1/lmm-auth-login.1 index 648f506..590e59e 100644 --- a/docs/man/man1/lmm-auth-login.1 +++ b/docs/man/man1/lmm-auth-login.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-auth-login - Authenticate with a mod source diff --git a/docs/man/man1/lmm-auth-logout.1 b/docs/man/man1/lmm-auth-logout.1 index 528f7a7..601c41e 100644 --- a/docs/man/man1/lmm-auth-logout.1 +++ b/docs/man/man1/lmm-auth-logout.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-auth-logout - Remove stored credentials for a mod source diff --git a/docs/man/man1/lmm-auth-status.1 b/docs/man/man1/lmm-auth-status.1 index 5015db7..0d42bf7 100644 --- a/docs/man/man1/lmm-auth-status.1 +++ b/docs/man/man1/lmm-auth-status.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-auth-status - Show authentication status for all sources diff --git a/docs/man/man1/lmm-auth.1 b/docs/man/man1/lmm-auth.1 index ffe3dc8..9c384aa 100644 --- a/docs/man/man1/lmm-auth.1 +++ b/docs/man/man1/lmm-auth.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-auth - Manage authentication for mod sources diff --git a/docs/man/man1/lmm-completion-bash.1 b/docs/man/man1/lmm-completion-bash.1 index e531e2b..910ced9 100644 --- a/docs/man/man1/lmm-completion-bash.1 +++ b/docs/man/man1/lmm-completion-bash.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-completion-bash - Generate the autocompletion script for bash diff --git a/docs/man/man1/lmm-completion-fish.1 b/docs/man/man1/lmm-completion-fish.1 index bd0e093..f3cb121 100644 --- a/docs/man/man1/lmm-completion-fish.1 +++ b/docs/man/man1/lmm-completion-fish.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-completion-fish - Generate the autocompletion script for fish diff --git a/docs/man/man1/lmm-completion-powershell.1 b/docs/man/man1/lmm-completion-powershell.1 index 97294ba..a18117f 100644 --- a/docs/man/man1/lmm-completion-powershell.1 +++ b/docs/man/man1/lmm-completion-powershell.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-completion-powershell - Generate the autocompletion script for powershell diff --git a/docs/man/man1/lmm-completion-zsh.1 b/docs/man/man1/lmm-completion-zsh.1 index 130a8da..ca60452 100644 --- a/docs/man/man1/lmm-completion-zsh.1 +++ b/docs/man/man1/lmm-completion-zsh.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-completion-zsh - Generate the autocompletion script for zsh diff --git a/docs/man/man1/lmm-completion.1 b/docs/man/man1/lmm-completion.1 index b99a439..ec7196d 100644 --- a/docs/man/man1/lmm-completion.1 +++ b/docs/man/man1/lmm-completion.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-completion - Generate the autocompletion script for the specified shell diff --git a/docs/man/man1/lmm-conflicts.1 b/docs/man/man1/lmm-conflicts.1 index 03c5f9d..e346b16 100644 --- a/docs/man/man1/lmm-conflicts.1 +++ b/docs/man/man1/lmm-conflicts.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-conflicts - Show all file conflicts in the current profile diff --git a/docs/man/man1/lmm-deploy.1 b/docs/man/man1/lmm-deploy.1 index 5850346..469f990 100644 --- a/docs/man/man1/lmm-deploy.1 +++ b/docs/man/man1/lmm-deploy.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-deploy - Deploy mods to game directory diff --git a/docs/man/man1/lmm-game-add.1 b/docs/man/man1/lmm-game-add.1 index 10dc0d2..7614879 100644 --- a/docs/man/man1/lmm-game-add.1 +++ b/docs/man/man1/lmm-game-add.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-game-add - Add a game interactively diff --git a/docs/man/man1/lmm-game-clear-default.1 b/docs/man/man1/lmm-game-clear-default.1 index 54a4cfb..db5b92e 100644 --- a/docs/man/man1/lmm-game-clear-default.1 +++ b/docs/man/man1/lmm-game-clear-default.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-game-clear-default - Clear the default game setting diff --git a/docs/man/man1/lmm-game-detect.1 b/docs/man/man1/lmm-game-detect.1 index 883d46e..29b1609 100644 --- a/docs/man/man1/lmm-game-detect.1 +++ b/docs/man/man1/lmm-game-detect.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-game-detect - Detect Steam games and add them to config diff --git a/docs/man/man1/lmm-game-set-default.1 b/docs/man/man1/lmm-game-set-default.1 index ebf8b4e..4bf4b92 100644 --- a/docs/man/man1/lmm-game-set-default.1 +++ b/docs/man/man1/lmm-game-set-default.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-game-set-default - Set the default game diff --git a/docs/man/man1/lmm-game-show-default.1 b/docs/man/man1/lmm-game-show-default.1 index a372bdf..3088809 100644 --- a/docs/man/man1/lmm-game-show-default.1 +++ b/docs/man/man1/lmm-game-show-default.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-game-show-default - Show the current default game diff --git a/docs/man/man1/lmm-game.1 b/docs/man/man1/lmm-game.1 index ae0e3f0..9cd96c7 100644 --- a/docs/man/man1/lmm-game.1 +++ b/docs/man/man1/lmm-game.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-game - Game management commands diff --git a/docs/man/man1/lmm-import.1 b/docs/man/man1/lmm-import.1 index 7089959..4be706e 100644 --- a/docs/man/man1/lmm-import.1 +++ b/docs/man/man1/lmm-import.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-import - Import mods from local files or scan mod_path diff --git a/docs/man/man1/lmm-install.1 b/docs/man/man1/lmm-install.1 index 1a8725b..e64b778 100644 --- a/docs/man/man1/lmm-install.1 +++ b/docs/man/man1/lmm-install.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-install - Install a mod diff --git a/docs/man/man1/lmm-list.1 b/docs/man/man1/lmm-list.1 index f6670fe..4936c7d 100644 --- a/docs/man/man1/lmm-list.1 +++ b/docs/man/man1/lmm-list.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-list - List installed mods diff --git a/docs/man/man1/lmm-mod-disable.1 b/docs/man/man1/lmm-mod-disable.1 index ecd24d7..c878b71 100644 --- a/docs/man/man1/lmm-mod-disable.1 +++ b/docs/man/man1/lmm-mod-disable.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-mod-disable - Disable a mod without uninstalling diff --git a/docs/man/man1/lmm-mod-edit.1 b/docs/man/man1/lmm-mod-edit.1 index 7de9fa6..1e0261b 100644 --- a/docs/man/man1/lmm-mod-edit.1 +++ b/docs/man/man1/lmm-mod-edit.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-mod-edit - Edit mod details (name, version, author, source, ID) diff --git a/docs/man/man1/lmm-mod-enable.1 b/docs/man/man1/lmm-mod-enable.1 index 0b04368..ed84d9a 100644 --- a/docs/man/man1/lmm-mod-enable.1 +++ b/docs/man/man1/lmm-mod-enable.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-mod-enable - Enable a disabled mod diff --git a/docs/man/man1/lmm-mod-files.1 b/docs/man/man1/lmm-mod-files.1 index 7ebb8be..2d4a75e 100644 --- a/docs/man/man1/lmm-mod-files.1 +++ b/docs/man/man1/lmm-mod-files.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-mod-files - List files deployed by a mod diff --git a/docs/man/man1/lmm-mod-lock.1 b/docs/man/man1/lmm-mod-lock.1 index d00ec8b..a48975a 100644 --- a/docs/man/man1/lmm-mod-lock.1 +++ b/docs/man/man1/lmm-mod-lock.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-mod-lock - Lock a mod at its current or a specific version diff --git a/docs/man/man1/lmm-mod-set-update.1 b/docs/man/man1/lmm-mod-set-update.1 index 14ed727..bb5781f 100644 --- a/docs/man/man1/lmm-mod-set-update.1 +++ b/docs/man/man1/lmm-mod-set-update.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-mod-set-update - Set update policy for a mod diff --git a/docs/man/man1/lmm-mod-show.1 b/docs/man/man1/lmm-mod-show.1 index 3ac7539..ec65ae1 100644 --- a/docs/man/man1/lmm-mod-show.1 +++ b/docs/man/man1/lmm-mod-show.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-mod-show - Show mod details diff --git a/docs/man/man1/lmm-mod-unlock.1 b/docs/man/man1/lmm-mod-unlock.1 index ad305bd..88d172b 100644 --- a/docs/man/man1/lmm-mod-unlock.1 +++ b/docs/man/man1/lmm-mod-unlock.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-mod-unlock - Unlock a mod, restoring normal update behavior diff --git a/docs/man/man1/lmm-mod.1 b/docs/man/man1/lmm-mod.1 index 03236d7..f57ba9d 100644 --- a/docs/man/man1/lmm-mod.1 +++ b/docs/man/man1/lmm-mod.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-mod - Manage mod settings diff --git a/docs/man/man1/lmm-profile-apply.1 b/docs/man/man1/lmm-profile-apply.1 index 47dea3a..e951f89 100644 --- a/docs/man/man1/lmm-profile-apply.1 +++ b/docs/man/man1/lmm-profile-apply.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-profile-apply - Apply profile to system diff --git a/docs/man/man1/lmm-profile-create.1 b/docs/man/man1/lmm-profile-create.1 index c20559d..fcff8f2 100644 --- a/docs/man/man1/lmm-profile-create.1 +++ b/docs/man/man1/lmm-profile-create.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-profile-create - Create a new profile diff --git a/docs/man/man1/lmm-profile-delete.1 b/docs/man/man1/lmm-profile-delete.1 index f443eed..88494ce 100644 --- a/docs/man/man1/lmm-profile-delete.1 +++ b/docs/man/man1/lmm-profile-delete.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-profile-delete - Delete a profile diff --git a/docs/man/man1/lmm-profile-export.1 b/docs/man/man1/lmm-profile-export.1 index 368eeec..1010096 100644 --- a/docs/man/man1/lmm-profile-export.1 +++ b/docs/man/man1/lmm-profile-export.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-profile-export - Export a profile diff --git a/docs/man/man1/lmm-profile-import.1 b/docs/man/man1/lmm-profile-import.1 index 830e9b2..0a40f8b 100644 --- a/docs/man/man1/lmm-profile-import.1 +++ b/docs/man/man1/lmm-profile-import.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-profile-import - Import a profile diff --git a/docs/man/man1/lmm-profile-list.1 b/docs/man/man1/lmm-profile-list.1 index a925377..7752284 100644 --- a/docs/man/man1/lmm-profile-list.1 +++ b/docs/man/man1/lmm-profile-list.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-profile-list - List all profiles diff --git a/docs/man/man1/lmm-profile-reorder.1 b/docs/man/man1/lmm-profile-reorder.1 index efe5d58..f2dbc27 100644 --- a/docs/man/man1/lmm-profile-reorder.1 +++ b/docs/man/man1/lmm-profile-reorder.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-profile-reorder - View or change load order diff --git a/docs/man/man1/lmm-profile-switch.1 b/docs/man/man1/lmm-profile-switch.1 index ce8fbab..1484abd 100644 --- a/docs/man/man1/lmm-profile-switch.1 +++ b/docs/man/man1/lmm-profile-switch.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-profile-switch - Switch to a different profile diff --git a/docs/man/man1/lmm-profile-sync.1 b/docs/man/man1/lmm-profile-sync.1 index b3c2d5d..84a0596 100644 --- a/docs/man/man1/lmm-profile-sync.1 +++ b/docs/man/man1/lmm-profile-sync.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-profile-sync - Sync profile to match installed mods diff --git a/docs/man/man1/lmm-profile.1 b/docs/man/man1/lmm-profile.1 index ee2a847..54b6925 100644 --- a/docs/man/man1/lmm-profile.1 +++ b/docs/man/man1/lmm-profile.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-profile - Manage mod profiles diff --git a/docs/man/man1/lmm-purge.1 b/docs/man/man1/lmm-purge.1 index c0944fd..3e67854 100644 --- a/docs/man/man1/lmm-purge.1 +++ b/docs/man/man1/lmm-purge.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-purge - Remove all deployed mods from game directory diff --git a/docs/man/man1/lmm-search.1 b/docs/man/man1/lmm-search.1 index c30b524..b96f163 100644 --- a/docs/man/man1/lmm-search.1 +++ b/docs/man/man1/lmm-search.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-search - Search for mods diff --git a/docs/man/man1/lmm-source-list.1 b/docs/man/man1/lmm-source-list.1 index 765a217..1811c43 100644 --- a/docs/man/man1/lmm-source-list.1 +++ b/docs/man/man1/lmm-source-list.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-source-list - List all mod sources diff --git a/docs/man/man1/lmm-source-validate.1 b/docs/man/man1/lmm-source-validate.1 index 47b2e70..0f00cb4 100644 --- a/docs/man/man1/lmm-source-validate.1 +++ b/docs/man/man1/lmm-source-validate.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-source-validate - Validate a source definition file diff --git a/docs/man/man1/lmm-source.1 b/docs/man/man1/lmm-source.1 index f40f771..c3cbcd1 100644 --- a/docs/man/man1/lmm-source.1 +++ b/docs/man/man1/lmm-source.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-source - Manage mod sources diff --git a/docs/man/man1/lmm-status.1 b/docs/man/man1/lmm-status.1 index 260183f..d67b523 100644 --- a/docs/man/man1/lmm-status.1 +++ b/docs/man/man1/lmm-status.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-status - Show current status diff --git a/docs/man/man1/lmm-tui.1 b/docs/man/man1/lmm-tui.1 index c00f4e9..5a1f2ff 100644 --- a/docs/man/man1/lmm-tui.1 +++ b/docs/man/man1/lmm-tui.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-tui - Launch the interactive terminal UI diff --git a/docs/man/man1/lmm-uninstall.1 b/docs/man/man1/lmm-uninstall.1 index 29ed395..d3b0368 100644 --- a/docs/man/man1/lmm-uninstall.1 +++ b/docs/man/man1/lmm-uninstall.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-uninstall - Uninstall a mod diff --git a/docs/man/man1/lmm-update-rollback.1 b/docs/man/man1/lmm-update-rollback.1 index 0a77405..bb9cc0c 100644 --- a/docs/man/man1/lmm-update-rollback.1 +++ b/docs/man/man1/lmm-update-rollback.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-update-rollback - Rollback a mod to its previous version diff --git a/docs/man/man1/lmm-update.1 b/docs/man/man1/lmm-update.1 index ecda796..eedbec9 100644 --- a/docs/man/man1/lmm-update.1 +++ b/docs/man/man1/lmm-update.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-update - Check for or apply mod updates diff --git a/docs/man/man1/lmm-verify.1 b/docs/man/man1/lmm-verify.1 index 2b065e4..f221a09 100644 --- a/docs/man/man1/lmm-verify.1 +++ b/docs/man/man1/lmm-verify.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm-verify - Verify cached mod files diff --git a/docs/man/man1/lmm.1 b/docs/man/man1/lmm.1 index 9191680..b0e1318 100644 --- a/docs/man/man1/lmm.1 +++ b/docs/man/man1/lmm.1 @@ -1,5 +1,5 @@ .nh -.TH "LMM" "1" "Jul 2026" "lmm 1.27.1" "User Commands" +.TH "LMM" "1" "Jul 2026" "lmm 1.28.0" "User Commands" .SH NAME lmm - Linux Mod Manager \- Terminal-based mod manager for Linux From 630d0e40a2055c85609af5b455229bd846d27781 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sun, 2 Aug 2026 00:40:06 -0400 Subject: [PATCH 91/96] fix: gameFromDetected fails loud on missing source config (#203 review) A known-games entry with neither Sources nor a non-empty NexusID silently produced {"nexusmods": ""} - a garbage source mapping that would propagate into games.yaml unnoticed. Every legitimate entry sets at least one of the two, so this is a misconfigured entry and now fails loud, naming the game, instead. The deploy_mode check still runs first (unchanged ordering), so an entry with both problems reports the deploy_mode error, matching the existing test's fixture. --- cmd/lmm/game.go | 25 +++++++++++++++--------- cmd/lmm/game_detect_test.go | 39 +++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/cmd/lmm/game.go b/cmd/lmm/game.go index 872b4a7..8c5d28f 100644 --- a/cmd/lmm/game.go +++ b/cmd/lmm/game.go @@ -254,21 +254,28 @@ func runGameDetect(cmd *cobra.Command, args []string) error { // wins outright; otherwise this derives the single-entry {nexusmods: // g.NexusID} map every detected game produced before Sources existed, so // every pre-#177 known game generates byte-for-byte the same games.yaml -// block it always has. g.DeployMode goes through domain.ParseDeployMode, -// which already treats "" as DeployExtract (today's default); an -// unrecognized non-empty value in the known-games schema (steam-games.yaml, -// built-in or user override) is a load-time error rather than a silent -// fallback (#172). +// block it always has. A known-games entry setting NEITHER is +// misconfigured - every legitimate entry sets at least one - and used to +// silently produce {"nexusmods": ""}, a garbage source mapping that would +// propagate into games.yaml unnoticed; that is now a fail-loud error naming +// the game instead (#203 release review). g.DeployMode goes through +// domain.ParseDeployMode, which already treats "" as DeployExtract (today's +// default); an unrecognized non-empty value in the known-games schema +// (steam-games.yaml, built-in or user override) is a load-time error rather +// than a silent fallback (#172). func gameFromDetected(g steam.DetectedGame) (*domain.Game, error) { - sources := g.Sources - if sources == nil { - sources = map[string]string{"nexusmods": g.NexusID} - } deployMode, ok := domain.ParseDeployMode(g.DeployMode) if !ok { return nil, fmt.Errorf("%w: steam-games.yaml: game %q: deploy_mode %q (valid: %s)", domain.ErrInvalidDeployMode, g.Slug, g.DeployMode, domain.ValidDeployModes) } + sources := g.Sources + if sources == nil { + if g.NexusID == "" { + return nil, fmt.Errorf("game %q: known-games entry has no sources and no nexus_id - set at least one", g.Slug) + } + sources = map[string]string{"nexusmods": g.NexusID} + } return &domain.Game{ ID: g.Slug, Name: g.Name, diff --git a/cmd/lmm/game_detect_test.go b/cmd/lmm/game_detect_test.go index 74eceed..f9abe44 100644 --- a/cmd/lmm/game_detect_test.go +++ b/cmd/lmm/game_detect_test.go @@ -92,6 +92,45 @@ func TestGameFromDetected_RejectsUnknownDeployMode(t *testing.T) { assert.Contains(t, err.Error(), "compil") } +// TestGameFromDetected_RequiresSourcesOrNexusID guards a Copilot release- +// review finding on #203: a known-games entry with neither Sources NOR a +// non-empty NexusID used to silently produce {"nexusmods": ""} - a garbage +// source mapping (an empty NexusMods game ID) that would propagate into +// games.yaml unnoticed. This is a misconfigured known-games entry (every +// legitimate one sets at least one of the two), so it must fail loud, +// naming the game, instead. Both legs: the failing case, and the two +// success cases (Sources-only, NexusID-only) that must remain unaffected. +func TestGameFromDetected_RequiresSourcesOrNexusID(t *testing.T) { + t.Run("neither Sources nor NexusID fails loud", func(t *testing.T) { + _, err := gameFromDetected(steam.DetectedGame{ + Slug: "misconfigured-game", + Name: "Misconfigured Game", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "misconfigured-game") + }) + + t.Run("NexusID alone is sufficient", func(t *testing.T) { + game, err := gameFromDetected(steam.DetectedGame{ + Slug: "skyrim-se", + Name: "Skyrim Special Edition", + NexusID: "skyrimspecialedition", + }) + require.NoError(t, err) + assert.Equal(t, map[string]string{"nexusmods": "skyrimspecialedition"}, game.SourceIDs) + }) + + t.Run("Sources alone is sufficient", func(t *testing.T) { + game, err := gameFromDetected(steam.DetectedGame{ + Slug: "icarus", + Name: "Icarus", + Sources: map[string]string{"icarus": "icarus"}, + }) + require.NoError(t, err) + assert.Equal(t, map[string]string{"icarus": "icarus"}, game.SourceIDs) + }) +} + // TestGameFromDetected_Icarus_ProducesReadmeEquivalentValues proves the // #177 acceptance criterion directly: saving a detected Icarus produces a // games.yaml entry whose values match the README's hand-written example From 09f238edd29e342e0145b3bb59ddf5c9ad42fac2 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sun, 2 Aug 2026 00:41:36 -0400 Subject: [PATCH 92/96] fix: name the URL and a capped body snippet in getJSON errors (#203 review) A non-200 response from Firestore produced a bare "HTTP %d" - a 403 was otherwise undiagnosable, since a permission error, a quota message, and a malformed request all look identical without knowing which request failed and what the server actually said. The error now includes the request URL and a 512-byte-capped snippet of the response body, read via io.LimitReader before continuing to drain the rest of the body to EOF (unchanged connection-reuse behavior - Close still happens on an already-drained body). --- internal/source/icarus/firestore_client.go | 18 ++++--- .../source/icarus/firestore_client_test.go | 50 +++++++++++++++++++ 2 files changed, 62 insertions(+), 6 deletions(-) diff --git a/internal/source/icarus/firestore_client.go b/internal/source/icarus/firestore_client.go index 11bb06f..c91dd4e 100644 --- a/internal/source/icarus/firestore_client.go +++ b/internal/source/icarus/firestore_client.go @@ -93,13 +93,19 @@ func (c *firestoreClient) getJSON(ctx context.Context, url string, out any) erro } defer resp.Body.Close() //nolint:errcheck if resp.StatusCode != http.StatusOK { - // Drain before Close so the underlying connection stays eligible for - // reuse — net/http only pools an HTTP/1.x connection once its body - // has been read to EOF; returning immediately here left whatever the - // server sent (however small) unread, forcing the transport to - // close the connection instead of reusing it for the next request. + // Read a capped snippet for the error message, then keep draining + // to EOF before Close so the underlying connection stays eligible + // for reuse — net/http only pools an HTTP/1.x connection once its + // body has been read to EOF; returning immediately here left + // whatever the server sent unread, forcing the transport to close + // the connection instead of reusing it for the next request. A bare + // "HTTP %d" left a 403 (or any other failure) undiagnosable: a + // Firestore permission error, a quota message, and a malformed + // request all look identical without the URL and body. + const errBodySnippetCap = 512 + snippet, _ := io.ReadAll(io.LimitReader(resp.Body, errBodySnippetCap)) _, _ = io.Copy(io.Discard, resp.Body) - return fmt.Errorf("HTTP %d", resp.StatusCode) + return fmt.Errorf("icarus: GET %s: HTTP %d: %s", url, resp.StatusCode, strings.TrimSpace(string(snippet))) } return json.NewDecoder(resp.Body).Decode(out) } diff --git a/internal/source/icarus/firestore_client_test.go b/internal/source/icarus/firestore_client_test.go index 4515288..93fb488 100644 --- a/internal/source/icarus/firestore_client_test.go +++ b/internal/source/icarus/firestore_client_test.go @@ -1,11 +1,13 @@ package icarus import ( + "bytes" "context" "encoding/json" "io" "net/http" "net/http/httptest" + "strings" "testing" ) @@ -215,3 +217,51 @@ func TestFirestoreClient_GetJSON_DrainsBodyBeforeCloseOnErrorPath(t *testing.T) t.Error("response body was not closed") } } + +// TestFirestoreClient_GetJSON_ErrorNamesURLAndBodySnippet guards a Copilot +// release-review finding on #203: getJSON's non-200 error was a bare "HTTP +// %d", naming neither which request failed nor what the server actually +// said - a 403 was otherwise undiagnosable (a Firestore permission error, a +// quota message, and a malformed-request response all look identical). The +// error must name the request URL and a snippet of the response body. +func TestFirestoreClient_GetJSON_ErrorNamesURLAndBodySnippet(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + w.Write([]byte(`{"error":{"message":"PERMISSION_DENIED: quota exceeded"}}`)) //nolint:errcheck + })) + defer srv.Close() + + c := newFirestoreClient("test-project", srv.Client()) + err := c.getJSON(context.Background(), srv.URL+"/some/path", &struct{}{}) + if err == nil { + t.Fatal("expected an error for a 403 response, got nil") + } + if !strings.Contains(err.Error(), srv.URL+"/some/path") { + t.Errorf("error %q does not name the request URL", err.Error()) + } + if !strings.Contains(err.Error(), "PERMISSION_DENIED") { + t.Errorf("error %q does not include the response body snippet", err.Error()) + } +} + +// TestFirestoreClient_GetJSON_ErrorBodySnippetIsCapped guards the "cap it" +// half of the same finding: a large/pathological error body (a stray HTML +// error page, a runaway response, ...) must not be echoed in full into the +// error message. +func TestFirestoreClient_GetJSON_ErrorBodySnippetIsCapped(t *testing.T) { + const bodySize = 64 * 1024 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + w.Write(bytes.Repeat([]byte("x"), bodySize)) //nolint:errcheck + })) + defer srv.Close() + + c := newFirestoreClient("test-project", srv.Client()) + err := c.getJSON(context.Background(), srv.URL, &struct{}{}) + if err == nil { + t.Fatal("expected an error for a 403 response, got nil") + } + if len(err.Error()) >= bodySize { + t.Errorf("error message is %d bytes - the response body snippet must be capped, not echoed in full", len(err.Error())) + } +} From 898513e85123a79836409be7cc1942aae6bd80a4 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sun, 2 Aug 2026 00:43:30 -0400 Subject: [PATCH 93/96] fix: cap total bundled-asset size in ParseExmodz (#203 review) The per-entry cap (maxZipEntrySize, 64 MiB) doesn't bound an archive with many entries each individually under it - five 60 MiB entries sum to 300 MiB despite none tripping the per-entry check. ParseExmodz now sums every asset entry's DECLARED uncompressed size up front and refuses the whole archive, naming it, before reading any asset content at all, if the combined total exceeds a new 256 MiB cap (maxZipTotalAssetsSize). The per-entry cap and the lying-declared-size read guard are unchanged. --- internal/source/icarus/exmodz.go | 34 +++++++++++++++--- internal/source/icarus/exmodz_test.go | 52 +++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 5 deletions(-) diff --git a/internal/source/icarus/exmodz.go b/internal/source/icarus/exmodz.go index c666de3..85bba03 100644 --- a/internal/source/icarus/exmodz.go +++ b/internal/source/icarus/exmodz.go @@ -62,22 +62,38 @@ func ParseExmodz(zipData []byte) (*ExmodzBundle, error) { if err != nil { return nil, fmt.Errorf("icarus: %s: %w", manifestPath, err) } - bundle := &ExmodzBundle{Diff: diff, Assets: make(map[string][]byte)} - + // Collect the asset entries first and sum their DECLARED sizes before + // reading any of them: the per-entry cap alone lets an archive with many + // entries, each individually under it, still declare an unbounded total + // (e.g. five 60 MiB entries sum to 300 MiB despite none tripping the + // 64 MiB per-entry check on its own). Checking the declared-size total + // up front refuses a pathological archive before any asset content is + // read into memory at all. + var assetFiles []*zip.File + var totalDeclaredSize uint64 for _, f := range zr.File { if f.Name == manifestPath || f.FileInfo().IsDir() { continue } - normalized := normalizeZipName(f.Name) - lower := strings.ToLower(normalized) + lower := strings.ToLower(normalizeZipName(f.Name)) if !strings.HasSuffix(lower, ".uasset") && !strings.HasSuffix(lower, ".uexp") { continue // skip readme/image/other non-asset files — never placed into the output pak } + assetFiles = append(assetFiles, f) + totalDeclaredSize += f.UncompressedSize64 + } + if totalDeclaredSize > maxZipTotalAssetsSize { + return nil, fmt.Errorf("icarus: .EXMODZ bundled assets declare a combined %d-byte uncompressed size, "+ + "exceeding the %d-byte total cap", totalDeclaredSize, uint64(maxZipTotalAssetsSize)) + } + + bundle := &ExmodzBundle{Diff: diff, Assets: make(map[string][]byte)} + for _, f := range assetFiles { data, err := readZipFile(f) if err != nil { return nil, fmt.Errorf("icarus: reading asset %s: %w", f.Name, err) } - bundle.Assets[normalized] = data + bundle.Assets[normalizeZipName(f.Name)] = data } return bundle, nil @@ -99,6 +115,14 @@ func normalizeZipName(name string) string { // UncompressedSize64 in a third-party, user-downloaded zip archive. const maxZipEntrySize = 64 << 20 +// maxZipTotalAssetsSize caps the COMBINED declared uncompressed size of +// every bundled asset in a single .EXMODZ - the per-entry cap alone doesn't +// bound an archive with many entries each individually under it (#203 +// release review). A real bundle's assets are a handful of small UE files; +// 256 MiB leaves generous headroom while still refusing to trust an +// archive that declares an unbounded total. +const maxZipTotalAssetsSize = 256 << 20 + func readZipFile(f *zip.File) ([]byte, error) { if f.UncompressedSize64 > maxZipEntrySize { return nil, fmt.Errorf("icarus: zip entry %s declares a %d-byte uncompressed size, "+ diff --git a/internal/source/icarus/exmodz_test.go b/internal/source/icarus/exmodz_test.go index d8af9ab..4423081 100644 --- a/internal/source/icarus/exmodz_test.go +++ b/internal/source/icarus/exmodz_test.go @@ -3,6 +3,7 @@ package icarus import ( "archive/zip" "bytes" + "fmt" "hash/crc32" "strings" "testing" @@ -280,3 +281,54 @@ func TestParseExmodz_RejectsLyingAssetDeclaredSize(t *testing.T) { t.Errorf("error %q should name the offending entry", err) } } + +// TestParseExmodz_RejectsOversizedTotalAssetsSize guards a Copilot release- +// review finding on #203: the per-entry cap (maxZipEntrySize, 64 MiB) alone +// lets an archive with MANY entries, each individually under that cap, +// still declare an unbounded total - e.g. five entries at 60 MiB apiece sum +// to 300 MiB despite no single entry tripping the per-entry check. A total +// cap across every bundled asset closes that gap. zw.CreateRaw declares the +// size fields verbatim, so the fixture never needs real hundreds of MiB of +// content, and the check must fire against the DECLARED sizes before any +// asset is read - the fixture's real content is just a few bytes. +func TestParseExmodz_RejectsOversizedTotalAssetsSize(t *testing.T) { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + + w, err := zw.Create("Extracted Mods/X.EXMOD") + if err != nil { + t.Fatal(err) + } + w.Write([]byte(`{"name":"X","Rows":[]}`)) //nolint:errcheck + + // 5 entries * 60 MiB declared = 300 MiB, over the 256 MiB total cap, + // while each individual entry (60 MiB) stays under the 64 MiB per-entry + // cap. + const perEntryDeclared = 60 << 20 + content := []byte("tiny") + for i := 0; i < 5; i++ { + rawW, err := zw.CreateRaw(&zip.FileHeader{ + Name: fmt.Sprintf("Bear_Mount/asset%d.uasset", i), + Method: zip.Store, + UncompressedSize64: perEntryDeclared, + CompressedSize64: uint64(len(content)), + CRC32: crc32.ChecksumIEEE(content), + }) + if err != nil { + t.Fatal(err) + } + rawW.Write(content) //nolint:errcheck + } + + if err := zw.Close(); err != nil { + t.Fatal(err) + } + + _, err = ParseExmodz(buf.Bytes()) + if err == nil { + t.Fatal("expected an error for bundled assets whose combined declared size exceeds the total cap, got nil") + } + if !strings.Contains(err.Error(), "EXMODZ") { + t.Errorf("error %q should name the archive", err) + } +} From d641be4c9920bafb7add12aef24a6f37b4dcf3fc Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sun, 2 Aug 2026 00:46:50 -0400 Subject: [PATCH 94/96] fix: thread merged-pak staleness reason into TUI UpdateItem (#203 review) The TUI hardcoded "(base pak updated)" for every RecompileNeeded row, even though core.CheckMergedPakStaleness already distinguishes two causes (RecompileReason: "base pak updated" - the fingerprint changed - or "not deployed" - the fingerprint matches but the artifact is missing) and lmm verify already shows the distinct reason. UpdateItem gains a RecompileReason field, threaded through from domain.Update in coreProvider.CheckUpdates; VersionLabel renders "()" instead of the hardcoded string, falling back to "base pak updated" only if RecompileReason is ever left empty (defensive - core always sets one). --- internal/tui/actions_provider.go | 30 +++++++-- internal/tui/service_core.go | 1 + internal/tui/service_core_recompile_test.go | 71 +++++++++++++++++++++ 3 files changed, 95 insertions(+), 7 deletions(-) diff --git a/internal/tui/actions_provider.go b/internal/tui/actions_provider.go index 45cf63d..a6a6378 100644 --- a/internal/tui/actions_provider.go +++ b/internal/tui/actions_provider.go @@ -288,18 +288,34 @@ type UpdateItem struct { // installed mod - and ApplyUpdate routes such a row to // Service.ApplyMergedPakRegen instead of Service.ApplyUpdate. RecompileNeeded bool + // RecompileReason is domain.Update's own distinct staleness reason + // ("base pak updated" - the fingerprint changed - or "not deployed" - + // the fingerprint still matches but the artifact is missing from the + // game directory), meaningful only when RecompileNeeded is true. #203 + // release review: the TUI used to hardcode "(base pak updated)" for + // every staleness row regardless of the real cause, unlike `lmm verify` + // (cmd/lmm/verify.go), which already names the distinct reason. + RecompileReason string } // VersionLabel renders u's version change for display: the normal -// " → " arrow for a real update, or "(base pak updated)" for a -// #197 RecompileNeeded row, where FromVersion == ToVersion and an arrow -// would misleadingly read as a no-op. Used everywhere an UpdateItem's -// version change is shown - the apply-updates modal, its result lines, and -// the changelog picker/overlay - so all of them read sanely for a -// staleness row without duplicating this branch four times. +// " → " arrow for a real update, or "()" for a #197 +// RecompileNeeded row, where FromVersion == ToVersion and an arrow would +// misleadingly read as a no-op - RecompileReason names the actual cause +// ("base pak updated" or "not deployed"), matching what `lmm verify` +// already shows; an empty RecompileReason (defensive - core always sets +// one alongside RecompileNeeded) falls back to "base pak updated" rather +// than rendering an empty "()". Used everywhere an UpdateItem's version +// change is shown - the apply-updates modal, its result lines, and the +// changelog picker/overlay - so all of them read sanely for a staleness +// row without duplicating this branch four times. func (u UpdateItem) VersionLabel() string { if u.RecompileNeeded { - return "(base pak updated)" + reason := u.RecompileReason + if reason == "" { + reason = "base pak updated" + } + return fmt.Sprintf("(%s)", reason) } return fmt.Sprintf("%s → %s", u.FromVersion, u.ToVersion) } diff --git a/internal/tui/service_core.go b/internal/tui/service_core.go index 6a37a38..119633b 100644 --- a/internal/tui/service_core.go +++ b/internal/tui/service_core.go @@ -1454,6 +1454,7 @@ func (p *coreProvider) CheckUpdates(ctx context.Context) (UpdatesView, error) { Changelog: core.CleanChangelog(u.Changelog), Locked: isLocked, LockedVersion: lockedVersion, RecompileNeeded: u.RecompileNeeded, + RecompileReason: u.RecompileReason, }) } if skipped := updateSkipWarning(core.CountUpdateSkips(installed)); skipped != "" { diff --git a/internal/tui/service_core_recompile_test.go b/internal/tui/service_core_recompile_test.go index e0ffe0d..6091637 100644 --- a/internal/tui/service_core_recompile_test.go +++ b/internal/tui/service_core_recompile_test.go @@ -145,9 +145,80 @@ func TestCoreProviderActions_CheckUpdates_ReportsRecompileNeeded(t *testing.T) { require.Equal(t, "merged-pak", u.ID) require.True(t, u.RecompileNeeded) require.Equal(t, u.FromVersion, u.ToVersion, "a staleness row has no real version change") + require.Equal(t, "base pak updated", u.RecompileReason, "a never-before-synced profile's reason is the fingerprint-mismatch default") require.Equal(t, "(base pak updated)", u.VersionLabel()) } +// TestCoreProviderActions_CheckUpdates_ReportsNotDeployedReason guards a +// #203 release-review finding: the TUI used to hardcode "(base pak +// updated)" for every staleness row, even when the real cause (the +// fingerprint still matches; the artifact is just missing from the game +// directory - core.CheckMergedPakStaleness's own "not deployed" case, +// internal/core/merged_pak.go) is different. After a real sync/deploy, the +// merged pak is removed WITHOUT anything else changing - the fingerprint +// still matches, so the distinct "not deployed" reason must surface, not +// the generic mismatch wording `lmm verify` would never use here either. +func TestCoreProviderActions_CheckUpdates_ReportsNotDeployedReason(t *testing.T) { + actions, _, deployedPath := newRecompileActionsFixture(t) + + view, err := actions.CheckUpdates(context.Background()) + require.NoError(t, err) + require.Len(t, view.Updates, 1) + _, err = actions.ApplyUpdate(context.Background(), view.Updates[0], nil) + require.NoError(t, err) + require.FileExists(t, deployedPath, "precondition: the merged pak must be deployed") + + require.NoError(t, os.Remove(deployedPath)) + + view, err = actions.CheckUpdates(context.Background()) + require.NoError(t, err) + require.Len(t, view.Updates, 1) + u := view.Updates[0] + require.True(t, u.RecompileNeeded) + require.Equal(t, "not deployed", u.RecompileReason) + require.Equal(t, "(not deployed)", u.VersionLabel()) +} + +// TestUpdateItem_VersionLabel is a pure unit test over VersionLabel's own +// branches, independent of any real staleness fixture: the normal arrow +// case, both real RecompileReason values core.CheckMergedPakStaleness can +// produce, and the defensive empty-reason fallback (core always sets one +// alongside RecompileNeeded, but VersionLabel must not render a bare "()" +// if some future caller ever leaves it unset). +func TestUpdateItem_VersionLabel(t *testing.T) { + tests := []struct { + name string + item tui.UpdateItem + want string + }{ + { + name: "normal update shows the version arrow", + item: tui.UpdateItem{FromVersion: "1.0", ToVersion: "2.0"}, + want: "1.0 → 2.0", + }, + { + name: "recompile needed: base pak updated", + item: tui.UpdateItem{RecompileNeeded: true, RecompileReason: "base pak updated"}, + want: "(base pak updated)", + }, + { + name: "recompile needed: not deployed", + item: tui.UpdateItem{RecompileNeeded: true, RecompileReason: "not deployed"}, + want: "(not deployed)", + }, + { + name: "recompile needed: empty reason falls back to base pak updated", + item: tui.UpdateItem{RecompileNeeded: true}, + want: "(base pak updated)", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.want, tt.item.VersionLabel()) + }) + } +} + // TestCoreProviderActions_ApplyUpdate_Recompile_AppliesAndRedeploys proves // ApplyUpdate dispatches a RecompileNeeded row to Service.ApplyMergedPakRegen // (checking RecompileNeeded BEFORE resolving a real InstalledMod - see From edc77615c417056a745c3249793a523f026b33cf Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sun, 2 Aug 2026 00:50:11 -0400 Subject: [PATCH 95/96] fix: doList surfaces LoadProfile errors beyond ErrProfileNotFound (#203 review) The lock-state lookup ignored EVERY config.LoadProfile error (`profileYAML, _ := ...`), including #172's fail-loud link_method validation - an invalid profile YAML silently degraded `lmm list` (no lock info, and per #201 every mod reading as "absent from the load order") instead of surfacing the same error every other command honors. Only domain.ErrProfileNotFound (a profile with no YAML on disk yet) is still tolerated; any other error, including validation, now aborts the listing. --- cmd/lmm/list.go | 20 ++++++--- cmd/lmm/list_profile_error_test.go | 65 ++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 5 deletions(-) create mode 100644 cmd/lmm/list_profile_error_test.go diff --git a/cmd/lmm/list.go b/cmd/lmm/list.go index 680d035..01c2976 100644 --- a/cmd/lmm/list.go +++ b/cmd/lmm/list.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "os" "text/tabwriter" @@ -96,11 +97,20 @@ func doList(cmd *cobra.Command, service *core.Service, game *domain.Game) error // #97: lock state lives on the profile YAML ref, not the DB row // GetInstalledMods above already returned - load it separately and key // by domain.ModKey for O(1) lookup per mod below (a precomputed map, - // not per-row FindRef, since this loops over every mod). Tolerant of a - // missing/unreadable profile.yaml (mirrors coreProvider.Overview's own - // "_ =" shape, internal/tui/service_core.go): an unreadable profile just - // means nothing shows as locked, not a failed listing. - profileYAML, _ := config.LoadProfile(service.ConfigDir(), game.ID, profileName) + // not per-row FindRef, since this loops over every mod). Tolerant ONLY + // of a genuinely missing profile.yaml (domain.ErrProfileNotFound) - a + // fresh profile with no YAML on disk yet just means nothing shows as + // locked, not a failed listing. Any OTHER LoadProfile error, including + // #172's fail-loud link_method validation, must surface immediately + // instead of silently degrading: swallowing it here used to mean an + // invalid profile YAML made `lmm list` quietly show no lock info AND + // (per #201) treat every mod as absent from the load order, instead of + // reporting the same error every other command honors (#203 release + // review). + profileYAML, err := config.LoadProfile(service.ConfigDir(), game.ID, profileName) + if err != nil && !errors.Is(err, domain.ErrProfileNotFound) { + return fmt.Errorf("loading profile: %w", err) + } lockedByKey := map[string]domain.ModReference{} if profileYAML != nil { for _, ref := range profileYAML.Mods { diff --git a/cmd/lmm/list_profile_error_test.go b/cmd/lmm/list_profile_error_test.go new file mode 100644 index 0000000..cb22681 --- /dev/null +++ b/cmd/lmm/list_profile_error_test.go @@ -0,0 +1,65 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/DonovanMods/linux-mod-manager/internal/domain" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestDoList_InvalidProfileLinkMethod_FailsLoud guards a Copilot release- +// review finding on #203: doList's own lock-state lookup used to ignore +// EVERY config.LoadProfile error (`profileYAML, _ := config.LoadProfile(...)`), +// including #172's fail-loud validation errors - an invalid link_method in +// the profile YAML would silently degrade `lmm list` (no lock info shown, +// and per #201 every mod would read as "absent from the load order") +// instead of surfacing the same load-time error every other command +// honors. Only a genuinely missing profile.yaml (domain.ErrProfileNotFound) +// is tolerable; any other error, including validation, must surface. +func TestDoList_InvalidProfileLinkMethod_FailsLoud(t *testing.T) { + svc, game := setupDoDeployTest(t) + oldListProfile, oldListProfiles := listProfile, listProfiles + listProfile, listProfiles = "", false + t.Cleanup(func() { listProfile, listProfiles = oldListProfile, oldListProfiles }) + seedDeployableMod(t, svc, game, "a", "Mod A", "a.esp") + + profilePath := filepath.Join(svc.ConfigDir(), "games", game.ID, "profiles", "default.yaml") + require.FileExists(t, profilePath, "precondition: setupDoDeployTest/seedDeployableMod must have created the profile") + data, err := os.ReadFile(profilePath) + require.NoError(t, err) + require.NoError(t, os.WriteFile(profilePath, append(data, []byte("\nlink_method: bogus\n")...), 0o644)) + + err = doList(&cobra.Command{}, svc, game) + require.Error(t, err) + assert.Contains(t, err.Error(), "link_method") + assert.Contains(t, err.Error(), "bogus") +} + +// TestDoList_MissingProfileYAML_StillLists is the tolerant leg's guard: a +// mod installed with NO profile.yaml ever written (e.g. a fresh DB row +// with no matching ProfileManager.Create/AddMod call) must still list +// successfully - only domain.ErrProfileNotFound is swallowed, unlike a +// genuine validation error (the sibling test above). +func TestDoList_MissingProfileYAML_StillLists(t *testing.T) { + svc, game := setupDoDeployTest(t) + oldListProfile, oldListProfiles := listProfile, listProfiles + listProfile, listProfiles = "", false + t.Cleanup(func() { listProfile, listProfiles = oldListProfile, oldListProfiles }) + + require.NoError(t, svc.SaveInstalledMod(&domain.InstalledMod{ + Mod: domain.Mod{ID: "a", SourceID: "src", Name: "Mod A", Version: "1.0", GameID: game.ID}, + ProfileName: "default", + UpdatePolicy: domain.UpdateNotify, + Enabled: true, + })) + profilePath := filepath.Join(svc.ConfigDir(), "games", game.ID, "profiles", "default.yaml") + _, statErr := os.Stat(profilePath) + require.True(t, os.IsNotExist(statErr), "precondition: no profile.yaml should exist yet") + + out := listNonVerbose(t, svc, game) + assert.Contains(t, out, "Mod A") +} From 15602ee3cd104b965630df3d383bec5fa07578b3 Mon Sep 17 00:00:00 2001 From: "Donovan C. Young" Date: Sun, 2 Aug 2026 01:01:43 -0400 Subject: [PATCH 96/96] fix: three Copilot round-2 findings on PR #204 (#203 release review) - gameFromDetected: treat an empty-but-non-nil Sources map the same as nil (len(sources) == 0), since YAML's `sources: {}` unmarshals to an empty map, not nil. - ParseExmodz: make the bundled-assets total-size accumulation overflow-safe by checking each declared size against the remaining cap headroom before adding, instead of summing first and comparing after - a naive sum could wrap a uint64 given attacker-controlled UncompressedSize64 values. - firestore_client.getJSON: omit the trailing body clause entirely when the trimmed error snippet is empty, avoiding a dangling colon. --- cmd/lmm/game.go | 2 +- cmd/lmm/game_detect_test.go | 15 +++++ internal/source/icarus/exmodz.go | 15 +++-- internal/source/icarus/exmodz_test.go | 58 +++++++++++++++++++ internal/source/icarus/firestore_client.go | 8 ++- .../source/icarus/firestore_client_test.go | 27 +++++++++ 6 files changed, 118 insertions(+), 7 deletions(-) diff --git a/cmd/lmm/game.go b/cmd/lmm/game.go index 8c5d28f..1c14e5e 100644 --- a/cmd/lmm/game.go +++ b/cmd/lmm/game.go @@ -270,7 +270,7 @@ func gameFromDetected(g steam.DetectedGame) (*domain.Game, error) { domain.ErrInvalidDeployMode, g.Slug, g.DeployMode, domain.ValidDeployModes) } sources := g.Sources - if sources == nil { + if len(sources) == 0 { if g.NexusID == "" { return nil, fmt.Errorf("game %q: known-games entry has no sources and no nexus_id - set at least one", g.Slug) } diff --git a/cmd/lmm/game_detect_test.go b/cmd/lmm/game_detect_test.go index f9abe44..22acdb9 100644 --- a/cmd/lmm/game_detect_test.go +++ b/cmd/lmm/game_detect_test.go @@ -129,6 +129,21 @@ func TestGameFromDetected_RequiresSourcesOrNexusID(t *testing.T) { require.NoError(t, err) assert.Equal(t, map[string]string{"icarus": "icarus"}, game.SourceIDs) }) + + // #204 round-2 review: YAML unmarshaling an explicit `sources: {}` line + // produces map[string]string{} (empty, non-nil), which the original + // `sources == nil` check let slide through, still landing at + // {"nexusmods": ""} when NexusID is also unset. Empty must be treated + // identically to nil. + t.Run("empty-but-non-nil Sources map fails the same as nil", func(t *testing.T) { + _, err := gameFromDetected(steam.DetectedGame{ + Slug: "empty-sources-game", + Name: "Empty Sources Game", + Sources: map[string]string{}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "empty-sources-game") + }) } // TestGameFromDetected_Icarus_ProducesReadmeEquivalentValues proves the diff --git a/internal/source/icarus/exmodz.go b/internal/source/icarus/exmodz.go index 85bba03..fbd3b09 100644 --- a/internal/source/icarus/exmodz.go +++ b/internal/source/icarus/exmodz.go @@ -79,13 +79,20 @@ func ParseExmodz(zipData []byte) (*ExmodzBundle, error) { if !strings.HasSuffix(lower, ".uasset") && !strings.HasSuffix(lower, ".uexp") { continue // skip readme/image/other non-asset files — never placed into the output pak } + // Checked against the remaining cap headroom BEFORE adding, not by + // summing first and comparing after: a declared size is + // attacker-controlled and `totalDeclaredSize += f.UncompressedSize64` + // can overflow a uint64, wrapping the running total to a value that + // falsely passes the cap check (#204 release review round 2). + // totalDeclaredSize <= maxZipTotalAssetsSize is an invariant of every + // prior iteration, so this subtraction never underflows. + if f.UncompressedSize64 > maxZipTotalAssetsSize-totalDeclaredSize { + return nil, fmt.Errorf("icarus: .EXMODZ bundled assets declare a combined uncompressed size "+ + "exceeding the %d-byte total cap", uint64(maxZipTotalAssetsSize)) + } assetFiles = append(assetFiles, f) totalDeclaredSize += f.UncompressedSize64 } - if totalDeclaredSize > maxZipTotalAssetsSize { - return nil, fmt.Errorf("icarus: .EXMODZ bundled assets declare a combined %d-byte uncompressed size, "+ - "exceeding the %d-byte total cap", totalDeclaredSize, uint64(maxZipTotalAssetsSize)) - } bundle := &ExmodzBundle{Diff: diff, Assets: make(map[string][]byte)} for _, f := range assetFiles { diff --git a/internal/source/icarus/exmodz_test.go b/internal/source/icarus/exmodz_test.go index 4423081..2f7eefd 100644 --- a/internal/source/icarus/exmodz_test.go +++ b/internal/source/icarus/exmodz_test.go @@ -5,6 +5,7 @@ import ( "bytes" "fmt" "hash/crc32" + "math" "strings" "testing" ) @@ -332,3 +333,60 @@ func TestParseExmodz_RejectsOversizedTotalAssetsSize(t *testing.T) { t.Errorf("error %q should name the archive", err) } } + +// TestParseExmodz_TotalAssetsSizeAccumulationIsOverflowSafe guards a +// Copilot round-2 release-review finding on #204: totalDeclaredSize += +// f.UncompressedSize64 summed attacker-controlled declared sizes with no +// per-entry gate, so two entries whose declared sizes overflow a uint64 sum +// wrap the total to a value far below the 256 MiB cap - the total-cap check +// meant to reject the whole bundle before any asset is read silently +// passes, and the bundle only fails afterward (if at all) via the +// UNRELATED per-entry cap inside readZipFile, once assets are actually +// being opened. The fix must reject against the remaining cap headroom +// BEFORE adding each declared size, so the total-cap error - not the +// per-entry one - fires first, without ever calling readZipFile. +func TestParseExmodz_TotalAssetsSizeAccumulationIsOverflowSafe(t *testing.T) { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + + w, err := zw.Create("Extracted Mods/X.EXMOD") + if err != nil { + t.Fatal(err) + } + w.Write([]byte(`{"name":"X","Rows":[]}`)) //nolint:errcheck + + content := []byte("tiny") + crc := crc32.ChecksumIEEE(content) + // The sum of these two declared sizes overflows uint64 and wraps to 49 + // (math.MaxUint64-50 + 100 == math.MaxUint64+50, mod 2^64), even though + // the first entry alone already dwarfs the 256 MiB total cap. + sizes := []uint64{math.MaxUint64 - 50, 100} + for i, size := range sizes { + rawW, err := zw.CreateRaw(&zip.FileHeader{ + Name: fmt.Sprintf("Bear_Mount/huge%d.uasset", i), + Method: zip.Store, + UncompressedSize64: size, + CompressedSize64: uint64(len(content)), + CRC32: crc, + }) + if err != nil { + t.Fatal(err) + } + rawW.Write(content) //nolint:errcheck + } + + if err := zw.Close(); err != nil { + t.Fatal(err) + } + + _, err = ParseExmodz(buf.Bytes()) + if err == nil { + t.Fatal("expected an error for a declared-size sum that overflows uint64, got nil") + } + if !strings.Contains(err.Error(), "total cap") { + t.Errorf("error %q should name the total cap - an overflowed sum must not slip past it undetected", err) + } + if strings.Contains(err.Error(), "per-entry cap") { + t.Errorf("error %q fired from the per-entry check, meaning uint64 overflow bypassed the total-cap check", err) + } +} diff --git a/internal/source/icarus/firestore_client.go b/internal/source/icarus/firestore_client.go index c91dd4e..2cda622 100644 --- a/internal/source/icarus/firestore_client.go +++ b/internal/source/icarus/firestore_client.go @@ -103,9 +103,13 @@ func (c *firestoreClient) getJSON(ctx context.Context, url string, out any) erro // Firestore permission error, a quota message, and a malformed // request all look identical without the URL and body. const errBodySnippetCap = 512 - snippet, _ := io.ReadAll(io.LimitReader(resp.Body, errBodySnippetCap)) + snippetBytes, _ := io.ReadAll(io.LimitReader(resp.Body, errBodySnippetCap)) _, _ = io.Copy(io.Discard, resp.Body) - return fmt.Errorf("icarus: GET %s: HTTP %d: %s", url, resp.StatusCode, strings.TrimSpace(string(snippet))) + snippet := strings.TrimSpace(string(snippetBytes)) + if snippet == "" { + return fmt.Errorf("icarus: GET %s: HTTP %d", url, resp.StatusCode) + } + return fmt.Errorf("icarus: GET %s: HTTP %d: %s", url, resp.StatusCode, snippet) } return json.NewDecoder(resp.Body).Decode(out) } diff --git a/internal/source/icarus/firestore_client_test.go b/internal/source/icarus/firestore_client_test.go index 93fb488..5854e33 100644 --- a/internal/source/icarus/firestore_client_test.go +++ b/internal/source/icarus/firestore_client_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io" "net/http" "net/http/httptest" @@ -265,3 +266,29 @@ func TestFirestoreClient_GetJSON_ErrorBodySnippetIsCapped(t *testing.T) { t.Errorf("error message is %d bytes - the response body snippet must be capped, not echoed in full", len(err.Error())) } } + +// TestFirestoreClient_GetJSON_ErrorOmitsBodyClauseWhenSnippetIsEmpty guards +// a Copilot round-2 release-review finding on #204: a non-200 response with +// no body (or a whitespace-only one) left the ": %s" clause in the error +// message with nothing after the trailing colon (e.g. "icarus: GET +// http://...: HTTP 403: "). The body clause must be omitted entirely when +// the trimmed snippet is empty, not rendered with a dangling colon. +func TestFirestoreClient_GetJSON_ErrorOmitsBodyClauseWhenSnippetIsEmpty(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + })) + defer srv.Close() + + c := newFirestoreClient("test-project", srv.Client()) + err := c.getJSON(context.Background(), srv.URL, &struct{}{}) + if err == nil { + t.Fatal("expected an error for a 403 response, got nil") + } + if strings.HasSuffix(err.Error(), ":") || strings.HasSuffix(err.Error(), ": ") { + t.Errorf("error %q has a dangling colon with no body snippet after it", err.Error()) + } + want := fmt.Sprintf("icarus: GET %s: HTTP 403", srv.URL) + if err.Error() != want { + t.Errorf("error = %q, want %q", err.Error(), want) + } +}