The bulk-data index at {@code api.scryfall.com/bulk-data} is a small JSON file + * (~3 KB) listing available datasets and their CDN download URIs. Once we have the + * download URI, the actual data file is served from {@code data.scryfall.io} (CDN, + * no rate limit). + */ +public final class BulkDataFetcher { + + private static final String BULK_INDEX_URL = "https://api.scryfall.com/bulk-data"; + private static final int CONNECT_TIMEOUT = 10_000; + private static final int READ_TIMEOUT = 300_000; // 5 min for large downloads + + private BulkDataFetcher() {} + + /** + * Fetches the bulk-data index and returns the download URI for the + * {@code default_cards} dataset (one English entry per print, ~100 MB). + * Sufficient for patching edition files; prefer this over {@link #fetchAllCardsUri} + * for faster downloads. + */ + public static String fetchDefaultCardsUri() throws IOException { + System.err.println("Fetching Scryfall bulk-data index..."); + String json = fetchText(BULK_INDEX_URL); + String uri = parseDownloadUri(json, "default_cards"); + if (uri == null) { + throw new IOException("'default_cards' entry not found in Scryfall bulk-data index"); + } + System.err.println(" Found: " + uri); + return uri; + } + + /** + * Fetches the bulk-data index and returns the download URI for the + * {@code all_cards} dataset (every language, every art variant, ~2.5 GB). + */ + public static String fetchAllCardsUri() throws IOException { + System.err.println("Fetching Scryfall bulk-data index..."); + String json = fetchText(BULK_INDEX_URL); + String uri = parseDownloadUri(json, "all_cards"); + if (uri == null) { + throw new IOException("'all_cards' entry not found in Scryfall bulk-data index"); + } + System.err.println(" Found: " + uri); + return uri; + } + + /** + * Downloads {@code sourceUrl} to {@code dest}, printing progress every 50 MB. + */ + public static void downloadToFile(String sourceUrl, Path dest) throws IOException { + System.err.println("Downloading: " + sourceUrl); + System.err.println(" to: " + dest.toAbsolutePath()); + + URL url = new URL(sourceUrl); + HttpURLConnection conn = openConnection(url); + long total = conn.getContentLengthLong(); + long bytesRead = 0L; + long lastReport = 0L; + byte[] buf = new byte[65_536]; + + try (InputStream in = conn.getInputStream(); + OutputStream out = new BufferedOutputStream(new FileOutputStream(dest.toFile()))) { + int n; + while ((n = in.read(buf)) > 0) { + out.write(buf, 0, n); + bytesRead += n; + if (bytesRead - lastReport >= 50L << 20) { + lastReport = bytesRead; + String progress = total > 0 + ? String.format("%.0f%%", 100.0 * bytesRead / total) + : String.format("%.0f MB received", bytesRead / 1e6); + System.err.printf(" %.1f MB [%s]%n", bytesRead / 1e6, progress); + } + } + } finally { + conn.disconnect(); + } + System.err.printf(" Download complete: %.1f MB%n", bytesRead / 1e6); + } + + /** + * Parses the bulk-data index JSON and returns the {@code download_uri} for + * the entry whose {@code type} matches {@code targetType}. + */ + static String parseDownloadUri(String json, String targetType) throws IOException { + try (JsonReader reader = new JsonReader(new StringReader(json))) { + reader.beginObject(); + while (reader.hasNext()) { + if ("data".equals(reader.nextName())) { + reader.beginArray(); + while (reader.hasNext()) { + String uri = readEntry(reader, targetType); + if (uri != null) { + return uri; + } + } + reader.endArray(); + } else { + reader.skipValue(); + } + } + reader.endObject(); + } catch (IOException e) { + return null; + } + return null; + } + + private static String readEntry(JsonReader reader, String targetType) throws IOException { + String type = null; + String downloadUri = null; + reader.beginObject(); + while (reader.hasNext()) { + String name = reader.nextName(); + if ("type".equals(name)) { + type = reader.nextString(); + } else if ("download_uri".equals(name)) { + downloadUri = reader.nextString(); + } else { + reader.skipValue(); + } + } + reader.endObject(); + return targetType.equals(type) ? downloadUri : null; + } + + private static String fetchText(String urlStr) throws IOException { + URL url = new URL(urlStr); + HttpURLConnection conn = openConnection(url); + try { + byte[] data = conn.getInputStream().readAllBytes(); + return new String(data, StandardCharsets.UTF_8); + } finally { + conn.disconnect(); + } + } + + private static HttpURLConnection openConnection(URL url) throws IOException { + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestProperty("User-Agent", "forge-scryfall-uuid-map/1.0"); + conn.setConnectTimeout(CONNECT_TIMEOUT); + conn.setReadTimeout(READ_TIMEOUT); + conn.setInstanceFollowRedirects(true); + conn.connect(); + int code = conn.getResponseCode(); + if (code != HttpURLConnection.HTTP_OK) { + conn.disconnect(); + throw new IOException("HTTP " + code + " fetching " + url); + } + return conn; + } +} diff --git a/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CardRecord.java b/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CardRecord.java new file mode 100644 index 000000000000..58f2befc13ee --- /dev/null +++ b/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CardRecord.java @@ -0,0 +1,9 @@ +package forge.scryfall.uuidmap; + +public record CardRecord( + String setCode, + String collectorNumber, + String lang, + String frontUuid, + String backUuid +) {} diff --git a/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CardStreamParser.java b/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CardStreamParser.java new file mode 100644 index 000000000000..611998f9a80f --- /dev/null +++ b/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CardStreamParser.java @@ -0,0 +1,156 @@ +package forge.scryfall.uuidmap; + +import com.google.gson.stream.JsonReader; + +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.function.Consumer; + +/** + * Streams a Scryfall bulk JSON file (all_cards or default_cards) and emits + * a {@link CardRecord} for every card entry that carries image data. + * + *
Uses Gson's streaming {@link JsonReader} so the 2.5 GB file is never
+ * fully loaded into memory — only one object at a time is in heap.
+ */
+public final class CardStreamParser {
+
+ private CardStreamParser() {}
+
+ /**
+ * Parses {@code bulkFile} and calls {@code consumer} for every record with image data.
+ *
+ * @return number of records emitted to the consumer
+ */
+ public static long parse(Path bulkFile, Consumer URL format: {@code https://cards.scryfall.io/normal/front/4/e/{uuid}.jpg?timestamp}
+ *
+ * Parsing the UUID from the URL (rather than using the card's {@code id} field directly)
+ * correctly handles the two Secret Lair DFC cards where both faces share an artwork UUID
+ * that differs from the card's own {@code id}.
+ *
+ * Falls back to {@code cardId} for non-CDN URLs such as
+ * {@code errors.scryfall.com/soon.jpg} (placeholder for missing images).
+ */
+ static String uuidFromUrl(String url, String cardId) {
+ if (url == null || !url.contains("cards.scryfall.io")) {
+ return cardId;
+ }
+ int qmark = url.indexOf('?');
+ String path = qmark >= 0 ? url.substring(0, qmark) : url;
+ int slash = path.lastIndexOf('/');
+ String filename = slash >= 0 ? path.substring(slash + 1) : path;
+ int dot = filename.lastIndexOf('.');
+ return dot >= 0 ? filename.substring(0, dot) : filename;
+ }
+}
diff --git a/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CdnUuidJsonWriter.java b/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CdnUuidJsonWriter.java
new file mode 100644
index 000000000000..530a1d0a9744
--- /dev/null
+++ b/forge-scryfall-uuid-map/src/main/java/forge/scryfall/uuidmap/CdnUuidJsonWriter.java
@@ -0,0 +1,137 @@
+package forge.scryfall.uuidmap;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Comparator;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Reads a Scryfall bulk JSON export and writes one UUID JSON file per set to
+ * {@code outputDir/{setCode}.json}.
+ *
+ * Each file maps collector number to a per-language UUID map:
+ * This set-per-file layout lets the runtime fetch exactly one file per set on
+ * demand and cache it locally, rather than shipping ~115k individual files with
+ * the game distribution.
+ *
+ * Collector-number and language keys are written in sorted order rather than
+ * bulk-file insertion order, and a file is only touched on disk if its content
+ * actually changed. The Scryfall bulk export doesn't guarantee stable card
+ * ordering between snapshots, so without this, re-running against a newer
+ * snapshot would rewrite nearly every set file with the same data in a
+ * different key order — a huge, meaningless diff every time this tool runs.
+ * With it, a re-run against unchanged sets touches nothing.
+ */
+public final class CdnUuidJsonWriter {
+
+ private static final Comparator Reads a Scryfall bulk JSON file and writes one JSON file per set:
+ * Uses the {@code all_cards} Scryfall dataset by default (every language, every
+ * art variant, ~2.5 GB) so that per-language UUID entries are fully populated.
+ * Pass {@code --default-cards} to instead fetch {@code default_cards} (~100 MB,
+ * one entry per card in English or nearest language only).
+ *
+ * CDN URL formula (for reference):
+ * {@code https://cards.scryfall.io/{size}/{front|back}/{uuid[0]}/{uuid[1]}/{uuid}.jpg}
+ *
+ * Usage:
+ *
+ * {
+ * "1": {"en": "uuid"}
+ * "2": {"en": "uuid", "ja": "ja-uuid"}
+ * "A-40":{"en": ["front-uuid", "back-uuid"]}
+ * }
+ *
+ *
+ *
+ * {outputDir}/{setCode}.json → {"cn":{"en":"uuid","ja":"uuid",...}, ...}
+ *
+ *
+ *
+ * # Download all_cards from Scryfall and write to cdn_uuid (full multi-language)
+ * java -jar forge-scryfall-uuid-map.jar --output-dir path/to/res/cdn_uuid
+ *
+ * # Provide a pre-downloaded bulk file
+ * java -jar forge-scryfall-uuid-map.jar --bulk-file all-cards-20260608.json \
+ * --output-dir path/to/res/cdn_uuid
+ *
+ * # English-only, smaller download (~100 MB)
+ * java -jar forge-scryfall-uuid-map.jar --output-dir path/to/res/cdn_uuid --default-cards
+ *
+ */
+public final class Main {
+
+ public static void main(String[] args) throws Exception {
+ Path bulkFile = null;
+ Path outputDir = null;
+ boolean defaultCards = false;
+
+ for (int i = 0; i < args.length; i++) {
+ switch (args[i]) {
+ case "--bulk-file": bulkFile = Path.of(args[++i]); break;
+ case "--output-dir": outputDir = Path.of(args[++i]); break;
+ case "--default-cards": defaultCards = true; break;
+ case "--help": printUsage(); return;
+ default:
+ System.err.println("Unknown argument: " + args[i]);
+ printUsage();
+ System.exit(1);
+ }
+ }
+
+ if (outputDir == null) {
+ System.err.println("Error: --output-dir is required.");
+ printUsage();
+ System.exit(1);
+ }
+ Files.createDirectories(outputDir);
+
+ if (bulkFile == null) {
+ bulkFile = findLocalBulkFile();
+ if (bulkFile != null) {
+ System.err.println("Auto-detected: " + bulkFile);
+ }
+ }
+
+ if (bulkFile == null || !Files.exists(bulkFile)) {
+ if (defaultCards) {
+ System.err.println("Fetching 'default_cards' from Scryfall (~100 MB, English only).");
+ String uri = BulkDataFetcher.fetchDefaultCardsUri();
+ bulkFile = Path.of(uriFilename(uri));
+ BulkDataFetcher.downloadToFile(uri, bulkFile);
+ } else {
+ System.err.println("Fetching 'all_cards' from Scryfall (~2.5 GB, all languages).");
+ System.err.println("Use --default-cards for a smaller English-only download.");
+ String uri = BulkDataFetcher.fetchAllCardsUri();
+ bulkFile = Path.of(uriFilename(uri));
+ BulkDataFetcher.downloadToFile(uri, bulkFile);
+ }
+ }
+
+ System.err.println("Input: " + bulkFile.toAbsolutePath());
+ System.err.println("Output dir: " + outputDir.toAbsolutePath());
+
+ long startMs = System.currentTimeMillis();
+ long written = CdnUuidJsonWriter.write(bulkFile, outputDir);
+ long elapsedMs = System.currentTimeMillis() - startMs;
+
+ System.err.printf("%nDone: %,d card entries written in %.1f s%n", written, elapsedMs / 1000.0);
+ }
+
+ private static String uriFilename(String uri) {
+ String path = uri.contains("?") ? uri.substring(0, uri.indexOf('?')) : uri;
+ return path.substring(path.lastIndexOf('/') + 1);
+ }
+
+ private static Path findLocalBulkFile() {
+ try (Stream