Preserve Crystal source VRAM tile identities in imported atlases

LICENSE-ROXAS2712.md

@@ -0,0 +1,88 @@
+# Roxas2712 — Fork Contributions
+
+Version 1.0 · Effective 25 October 2026
+
+Copyright © 2026 Roxas2712, to the extent of rights held.
+All rights reserved except as expressly granted below or already granted.
+
+## 1. Scope and ownership
+
+This notice applies only to original contributions made by Roxas2712
+(GitHub: Roxas2712) to `Roxas2712/Gen2Recomped`, including original additions
+within modified files, insofar as he owns the relevant rights. Git history,
+pull-request diffs and source provenance identify contributions; a commit
+author or a whole-file change alone does not establish ownership of every
+part of that file.
+
+It does not claim ownership of Cedric's work, Gen1Recomp / BOIS CLUB GAMES
+work, other contributors' work, third-party dependencies, or ROM-derived
+content. Their applicable licences and notices remain in force. It does not
+grant permission to distribute the complete combined fork where another
+rights holder's permission is required.
+
+## 2. General permission and reserved rights
+
+For contributions covered by this notice and not already licensed more
+broadly, you may read and study the published source and make private,
+non-commercial modifications and builds for your own use. This grants no
+general permission to redistribute, sublicense, sell, or offer those
+contributions as a hosted service. Other uses require Roxas2712's
+written permission, except for the specific permission below and rights
+available independently under applicable law.
+
+Keep the relevant copyright, licence and attribution notices. Do not
+represent Roxas2712's contributions as exclusively authored by someone
+else. Public source availability alone does not put them in the public
+domain or grant an open-source licence.
+
+## 3. Specific permission for Cedric / UNDERdecoded
+
+Roxas2712 expressly grants Cedric (Ced; GitHub: UNDERdecoded /
+UNDERdecodedHD) a non-exclusive, worldwide, perpetual, royalty-free
+permission to use, reproduce, integrate, modify and distribute the covered
+contributions that Roxas2712 submits or expressly offers through pull
+requests to `UNDERdecoded/Gen2Recomped`, for Cedric's Gen2Recomped /
+RetroRecompiled project. This includes its renamed continuation maintained
+by Cedric, its official builds, and its commercial operation.
+
+Cedric may license the incorporated contributions to recipients as part of
+that project, subject to preserving this permission's applicable credit and
+notice requirements. This permission is not a copyright assignment, is not
+exclusive, and does not cover unpublished work, separate repositories,
+ModCompass service operation, credentials, or content not offered in those
+pull requests. Roxas2712 retains his own rights to his contributions.
+
+For uses relying on this permission:
+
+- retain the relevant source copyright and licence notices;
+- give visible credit in the project's README or equivalent public project
+  page, and in the application's existing About / Credits view, if present;
+- use at least the following credit, or a substantially equivalent one:
+
+  > Contributions by Roxas2712 —
+  > https://github.com/Roxas2712
+
+A maintained consolidated contributor credit is sufficient; a separate
+credit on every screen or for every change is not required. Adapted
+contributions must not be attributed exclusively to another author. Other
+contributors' credits must also remain intact.
+
+## 4. Earlier licences and permissions remain effective
+
+This notice does not revoke, narrow, or retrospectively add conditions to
+any permission already validly granted, including MIT / GPL grants,
+separate agreements, or permissions arising from previously submitted
+pull requests. Where an earlier grant permits a use independently of this
+notice, its existing terms govern that use. For such earlier uses, the
+visible credit above is requested to the extent it is not already required.
+
+For subsequent contributions, this is the default statement of Roxas2712's own reserved rights and the specific PR permission, subject to any
+applicable existing licence obligations or separately agreed terms. It
+does not change another rights holder's licence or guarantee compatibility
+of a particular combination of code.
+
+## 5. No warranty
+
+To the extent permitted by applicable law, these contributions are provided
+as is, without warranty. No support obligation, endorsement, or rights to
+third-party code, game assets, names, or trademarks are granted.

src/import/Gen2VramLayout.lua

@@ -0,0 +1,32 @@
+-- Copyright (c) 2026 Roxas2712. See LICENSE-ROXAS2712.md.
+-- Keep Crystal's public tile addresses stable across both VRAM banks.
+-- LoadTilesetGFX supplies 96 tiles per bank; bank 1 starts at tile $80.
+local Layout={REVISION=1,BANK_TILES=96,BANK_STRIDE=128,ATLAS_TILES=256}
+
+function Layout.expand(bytes)
+  if type(bytes)~='table' or #bytes~=96*2*16 then return bytes,nil end
+  local out={}
+  for i=1,256*16 do out[i]=0 end
+  for i=1,96*16 do
+    out[i]=bytes[i]
+    out[128*16+i]=bytes[96*16+i]
+  end
+  return out,128
+end
+
+function Layout.tileIndex(tile)
+  if type(tile)~='number' or tile%1~=0 then return 0 end
+  if tile>=0 and tile<96 or tile>=128 and tile<224 then return tile end
+  return 0
+end
+
+function Layout.paletteMap(nibbles)
+  local out={}
+  for tile=0,255 do
+    local valid=tile<96 or tile>=128 and tile<224
+    out[tile+1]=valid and (nibbles[tile+1]or 0)%8 or 0
+  end
+  return out
+end
+
+return Layout

src/import/RomExtractorGen2.lua

@@ -677,8 +677,10 @@ function GEN2_PAL.twoVramBanks(tileCount)
   return tileCount == GEN2_PAL.tilesPerVramBank * 2
 end
 
--- Metatile byte -> index into the flattened 0..191 sheet.
-function GEN2_PAL.tileIndex(byte, tileCount)
+-- Metatile byte -> extracted atlas index. New split-bank sheets retain the
+-- source $80 bank boundary; compact legacy/other formats keep their path.
+function GEN2_PAL.tileIndex(byte, tileCount, bankStride)
+  if bankStride==128 then return require('src.import.Gen2VramLayout').tileIndex(byte) end
   if not GEN2_PAL.twoVramBanks(tileCount) then return byte end
   if byte < GEN2_PAL.tilesPerVramBank then return byte end
   if byte >= 0x80 then
@@ -16262,11 +16264,20 @@ function RomExtractorGen2:extractRuntimeScaffolds()
       end
       blockCount = blockCount or 0x80
       blocksRaw = blocksRaw or self.rom:bytes(meta.bank, meta.address, blockCount * 16)
-      -- Flatten the two VRAM banks into one 0..191 range here, so every reader
-      -- downstream -- the atlas bake, the animated-tile overdraw, mods -- can
-      -- treat a metatile byte as a plain index into the extracted sheet.
-      -- See GEN2_PAL.tileIndex.
       local tileCount = (width / 8) * (height / 8)
+      -- The source address is part of a tile's identity: compacting bank 1
+      -- down to $60 changed the ids used by terrain/model readers. Reserve
+      -- the VRAM gap and keep the bytes, atlas and palette on the same grid.
+      -- Attribute-bank formats (Prism/Polished) keep their separate decoder.
+      local bankStride
+      if not preAttrRaw and GEN2_PAL.twoVramBanks(tileCount) then
+        pixels,bankStride=require('src.import.Gen2VramLayout').expand(pixels)
+        if bankStride then
+          width,height=inferTilesetDimensions(#pixels)
+          ImageWriter.save(ImageWriter.decode2bpp(pixels,width,height),imagePath)
+          tileCount=width*height/64
+        end
+      end
       local blocks = {}
       -- Prism splits its sheet the same way, at a different offset and by a
       -- different route.  LoadTileset (home/map.asm) zero-fills scratch to
@@ -16314,7 +16325,7 @@ function RomExtractorGen2:extractRuntimeScaffolds()
             index = flipped(index, attr)
             palMapFromAttr[index + 1] = attr % 8
           else
-            index = GEN2_PAL.tileIndex(blocksRaw[pos], tileCount)
+            index = GEN2_PAL.tileIndex(blocksRaw[pos], tileCount, bankStride)
           end
           block[#block + 1] = index
         end
@@ -16358,6 +16369,7 @@ function RomExtractorGen2:extractRuntimeScaffolds()
         imageWidth = width,
         imageHeight = height,
         tilesPerRow = width / 8,
+        gen2VramBankStride = bankStride,
         blocks = blocks,
         -- Set only where <Tileset>Attr supplied it (Prism).  The PalMap
         -- reader below sees this and leaves it alone rather than looking for
@@ -16408,7 +16420,8 @@ function RomExtractorGen2:extractRuntimeScaffolds()
       if palSym and not tilesets[id].palMap then
         -- How many tiles the sheet actually came out as decides both the
         -- palette map's length and whether it carries a VRAM-bank bit.
-        local twoBanks = GEN2_PAL.twoVramBanks(GEN2_PAL.sheetTiles(tilesets[id]))
+        local logicalBanks=tilesets[id].gen2VramBankStride==128
+        local twoBanks = logicalBanks or GEN2_PAL.twoVramBanks(GEN2_PAL.sheetTiles(tilesets[id]))
         local palOk, palRaw = pcall(function()
           return self.rom:bytes(palSym.bank, palSym.address,
             twoBanks and GEN2_PAL.palMapBytes or GEN2_PAL.palMapBytesOneBank)
@@ -16417,9 +16430,9 @@ function RomExtractorGen2:extractRuntimeScaffolds()
           -- Crystal's palette map is 224 nibbles -- one per metatile byte
           -- $00-$DF, each `vramBank << 3 | palette` -- not 96.  Reading only
           -- the first 96 stopped at the bank-0 half, so every second-bank tile
-          -- took a clamped, wrong palette.  Re-key it by the same flattened
-          -- 0..191 index GEN2_PAL.tileIndex produces and drop the bank bit, which
-          -- the block bytes now carry themselves.  Reading PAST the map would
+          -- took a clamped, wrong palette. Keep it aligned with the extracted
+          -- tile-address contract (logical or legacy compact), dropping the
+          -- bank bit from each palette nibble.  Reading PAST the map would
           -- run into the next tileset's, which is why the length is version-
           -- derived rather than fixed.
           local nibbles = {}
@@ -16428,7 +16441,9 @@ function RomExtractorGen2:extractRuntimeScaffolds()
             nibbles[#nibbles + 1] = math.floor(byte / 16)
           end
           local palMap = {}
-          if twoBanks then
+          if logicalBanks then
+            palMap=require('src.import.Gen2VramLayout').paletteMap(nibbles)
+          elseif twoBanks then
             for tile = 0, GEN2_PAL.tilesPerVramBank * 2 - 1 do
               local raw = tile < GEN2_PAL.tilesPerVramBank and tile
                 or (tile - GEN2_PAL.tilesPerVramBank + 0x80)

src/import/RomImporter.lua

@@ -1323,6 +1323,9 @@ local function markerFor(version)
   if version == "red" or version == "blue" or version == "yellow" then
     revision = "gen1-trade-art-v1:"
   end
+  if GameVersion.generation(version)==2 then
+    revision=revision..'gen2-vram-layout-v'..require('src.import.Gen2VramLayout').REVISION..':'
+  end
   return CACHE_FORMAT .. revision .. GameVersion.info(version).sha1
 end
 

tests/engine/gen2_vram_layout.lua

@@ -0,0 +1,127 @@
+-- Copyright (c) 2026 Roxas2712. See LICENSE-ROXAS2712.md.
+package.path='./?.lua;./?/init.lua;'..package.path
+local T=require('tests.modkit')
+local L=require('src.import.Gen2VramLayout')
+local E=require('src.import.RomExtractorGen2')
+local Writer=require('src.import.ImageWriter')
+local Version=require('src.core.GameVersion')
+local Importer=require('src.import.RomImporter')
+local before=Version.get()
+local bytes={};for i=1,3072 do bytes[i]=(i*17+math.floor(i/1537)*9)%256 end
+local expanded,stride=L.expand(bytes)
+T.eq(stride,128,'two 96-tile banks retain the $80 source address')
+T.eq(#expanded,4096,'both banks and reserved slots fit one 256-tile atlas')
+local bank0,bank1,gap,tail={},{},{},{}
+for i=1,1536 do bank0[i]=expanded[i];bank1[i]=expanded[2048+i]end
+T.same(bank0,{unpack(bytes,1,1536)},'all first-bank bytes preserved')
+T.same(bank1,{unpack(bytes,1537,3072)},'all second-bank bytes preserved')
+for i=1537,2048 do gap[#gap+1]=expanded[i]end
+for i=3585,4096 do tail[#tail+1]=expanded[i]end
+T.eq(table.concat(gap),string.rep('0',512),'reserved $60-$7f slots are blank')
+T.eq(table.concat(tail),string.rep('0',512),'unused $e0-$ff slots are blank')
+for _,n in ipairs({0,1536,4096,6144})do
+ local raw={};for i=1,n do raw[i]=i%256 end
+ local out,bank=L.expand(raw)
+ T.eq(out,raw,'single-bank, placeholder and extended layouts retain identity')
+ T.eq(bank,nil,'non-split payload cannot advertise bank 1')
+end
+local palettes={};for i=1,224 do palettes[i]=(i-1)%16 end
+local mapped=L.paletteMap(palettes)
+T.eq(#mapped,256,'palette addresses use the same dense atlas')
+for tile=0,255 do
+ local valid=tile<96 or tile>=128 and tile<224
+ T.eq(L.tileIndex(tile),valid and tile or 0,'source tile address '..tile)
+ T.eq(mapped[tile+1],valid and tile%8 or 0,'palette follows source tile '..tile)
+end
+for _,bad in ipairs({-1,256,1.5,'128',false})do T.eq(L.tileIndex(bad),0,'malformed tile cannot address the atlas')end
+
+-- Real extraction path with synthetic ROM bytes; only image encoding is
+-- captured, so there is no graphics driver or copyrighted cartridge fixture.
+local decode,save=Writer.decode2bpp,Writer.save
+local lastImage
+Writer.decode2bpp=function(raw,w,h)return{raw=raw,width=w,height=h}end
+Writer.save=function(image)lastImage=image;return true end
+local function fixture(version,tiles,attributes)
+ Version.set(version);lastImage=nil
+ local symbols={TilesetJohtoGFX={1,0x4000},TilesetJohtoMeta={1,0x5000},
+  TilesetJohtoColl={1,0x5020},TilesetJohtoPalMap={1,0x6000}}
+ local e=E.new(nil,version,{symbols=symbols,tilesets={TilesetJohto={blockCount=2}}})
+ local compressed={}
+ local function fill(n,value)
+  while n>0 do local count=math.min(n,1024);local length=count-1
+   compressed[#compressed+1]=0xe4+math.floor(length/256)
+   compressed[#compressed+1]=length%256;compressed[#compressed+1]=value;n=n-count
+  end
+ end
+ fill(96*16,0x3c);if tiles>96 then fill((tiles-96)*16,0xa5)end;compressed[#compressed+1]=0xff
+ local block={};for i=1,(attributes and 256 or 32)do block[i]=({0,95,128,159,191,223,224,255})[(i-1)%8+1]end
+ if attributes then for i=1,#block do block[i]=(i-1)%96 end end
+ local palette={};for i=1,112 do palette[i]=i<=48 and 0x10 or i<=64 and 0xff or 0xba end
+ e.rom={bytes=function(_,_,address,count)
+  if address==0x4000 then return compressed end
+  if address==0x5000 then return block end
+  if address==0x5020 then local r={};for i=1,count do r[i]=0 end;return r end
+  if address==0x6000 then local r={};for i=1,count do r[i]=palette[i]end;return r end
+  error('unexpected synthetic ROM read')
+ end}
+ e.constants=function()return{tilesetOrder={'TilesetJohto'}}end
+ e.readSourceTable=function()return{TEST={tileset='TilesetJohto'}}end
+ e.gen2EnvPalettes=function()return nil end;e.gen2TilesetEnvironments=function()return{}end
+ e.gen2CollisionClasses=function()return{land={},water={}}end
+ e.gen2TilesetAnimation=function()end
+ if attributes then
+  e.layout=function(_,key,default)if key=='tilesetCompressed'then return 1 end;return default end
+  e.gen2Lz=function(_,name)
+   if name=='TilesetJohtoMeta'then return block end
+   if name=='TilesetJohtoColl'then return{0,0,0,0}end
+   if name=='TilesetJohtoAttr'then local r={};for i=1,#block do r[i]=i%2==0 and 10 or 3 end;return r end
+  end
+ end
+ local stop,result={},nil
+ e.write=function(_,name,value)if name=='tilesets'then result=value;error(stop,0)end end
+ local ok,why=pcall(e.extractRuntimeScaffolds,e)
+ T.eq(ok,false,'fixture stops after the real tileset publication boundary')
+ T.eq(why,stop,'extractor reaches native tileset publication without other errors')
+ return result and result.TilesetJohto,lastImage
+end
+local split,img=fixture('crystal',192)
+T.eq(split.gen2VramBankStride,128,'real importer publishes logical-bank descriptor')
+T.eq(split.imageWidth,128,'atlas width stays stable')
+T.eq(split.imageHeight,128,'real image metadata matches expanded address space')
+T.eq(img.height,split.imageHeight,'saved image and definition use the same height')
+T.eq(img.raw[96*16+1],0,'saved reserved slot is blank')
+T.eq(img.raw[128*16+1],0xa5,'saved bank-1 pixels start at tile $80')
+T.same(split.blocks[1],{0,95,128,159,191,223,0,0,0,95,128,159,191,223,0,0},'actual metatile references retain source ids')
+T.eq(#split.palMap,256,'actual palette map retains dense logical addresses')
+T.eq(split.palMap[129],2,'actual bank-1 palette follows $80 tile')
+T.eq(split.palMap[130],3,'actual odd bank-1 tile selects its own palette')
+for _,edition in ipairs({'gold','silver'})do
+ local one,image=fixture(edition,96)
+ T.eq(one.gen2VramBankStride,nil,edition..' single bank is unchanged')
+ T.eq(one.imageHeight,48,edition..' native 96-tile dimensions retained')
+ T.eq(image.height,48,edition..' saved single-bank dimensions retained')
+end
+for _,edition in ipairs({'prism','polishedcrystal'})do
+ local attr,image=fixture(edition,256,true)
+ T.eq(attr.gen2VramBankStride,nil,edition..' attribute-bank layout is not mistaken for packed Crystal')
+ T.eq(attr.blocks[1][2],129,edition..' explicit attribute-bank reference retained')
+ T.eq(attr.palMap[130],2,edition..' attribute palette follows its original owner')
+end
+Writer.decode2bpp,Writer.save=decode,save
+
+local markerFor
+for i=1,100 do local name,value=debug.getupvalue(Importer.readyReport,i)
+ if name=='markerFor'then markerFor=value;break end;if not name then break end
+end
+assert(markerFor)
+for _,edition in ipairs({'gold','silver','crystal','prism','polishedcrystal'})do
+ local marker=markerFor(edition)
+ T.check(marker:find('gen2%-vram%-layout%-v1:')~=nil,edition..' cache records tile-address contract')
+ Version.set(edition);love.filesystem.write(Version.cachePrefix(edition)..'rom-cache.complete',marker:gsub('gen2%-vram%-layout%-v1:',''))
+ T.eq(Importer.readyReport(edition).ok,false,edition..' compact legacy cache cannot skip reimport')
+end
+for _,edition in ipairs({'red','emerald','platinum'})do
+ T.check(not markerFor(edition):find('gen2%-vram%-layout%-v1:'),edition..' unrelated import marker is unchanged')
+end
+Version.set(before)
+T.finish('native Crystal VRAM tile identity, extraction, palette alignment and cache invalidation')