|
| 1 | +package forge.scryfall.uuidmap; |
| 2 | + |
| 3 | +import com.google.gson.stream.JsonReader; |
| 4 | + |
| 5 | +import java.io.BufferedOutputStream; |
| 6 | +import java.io.FileOutputStream; |
| 7 | +import java.io.IOException; |
| 8 | +import java.io.InputStream; |
| 9 | +import java.io.OutputStream; |
| 10 | +import java.io.StringReader; |
| 11 | +import java.net.HttpURLConnection; |
| 12 | +import java.net.URL; |
| 13 | +import java.nio.charset.StandardCharsets; |
| 14 | +import java.nio.file.Path; |
| 15 | + |
| 16 | +/** |
| 17 | + * Fetches the Scryfall bulk-data index and downloads the {@code all_cards} dataset. |
| 18 | + * |
| 19 | + * <p>The bulk-data index at {@code api.scryfall.com/bulk-data} is a small JSON file |
| 20 | + * (~3 KB) listing available datasets and their CDN download URIs. Once we have the |
| 21 | + * download URI, the actual data file is served from {@code data.scryfall.io} (CDN, |
| 22 | + * no rate limit). |
| 23 | + */ |
| 24 | +public final class BulkDataFetcher { |
| 25 | + |
| 26 | + private static final String BULK_INDEX_URL = "https://api.scryfall.com/bulk-data"; |
| 27 | + private static final int CONNECT_TIMEOUT = 10_000; |
| 28 | + private static final int READ_TIMEOUT = 300_000; // 5 min for large downloads |
| 29 | + |
| 30 | + private BulkDataFetcher() {} |
| 31 | + |
| 32 | + /** |
| 33 | + * Fetches the bulk-data index and returns the download URI for the |
| 34 | + * {@code default_cards} dataset (one English entry per print, ~100 MB). |
| 35 | + * Sufficient for patching edition files; prefer this over {@link #fetchAllCardsUri} |
| 36 | + * for faster downloads. |
| 37 | + */ |
| 38 | + public static String fetchDefaultCardsUri() throws IOException { |
| 39 | + System.err.println("Fetching Scryfall bulk-data index..."); |
| 40 | + String json = fetchText(BULK_INDEX_URL); |
| 41 | + String uri = parseDownloadUri(json, "default_cards"); |
| 42 | + if (uri == null) { |
| 43 | + throw new IOException("'default_cards' entry not found in Scryfall bulk-data index"); |
| 44 | + } |
| 45 | + System.err.println(" Found: " + uri); |
| 46 | + return uri; |
| 47 | + } |
| 48 | + |
| 49 | + /** |
| 50 | + * Fetches the bulk-data index and returns the download URI for the |
| 51 | + * {@code all_cards} dataset (every language, every art variant, ~2.5 GB). |
| 52 | + */ |
| 53 | + public static String fetchAllCardsUri() throws IOException { |
| 54 | + System.err.println("Fetching Scryfall bulk-data index..."); |
| 55 | + String json = fetchText(BULK_INDEX_URL); |
| 56 | + String uri = parseDownloadUri(json, "all_cards"); |
| 57 | + if (uri == null) { |
| 58 | + throw new IOException("'all_cards' entry not found in Scryfall bulk-data index"); |
| 59 | + } |
| 60 | + System.err.println(" Found: " + uri); |
| 61 | + return uri; |
| 62 | + } |
| 63 | + |
| 64 | + /** |
| 65 | + * Downloads {@code sourceUrl} to {@code dest}, printing progress every 50 MB. |
| 66 | + */ |
| 67 | + public static void downloadToFile(String sourceUrl, Path dest) throws IOException { |
| 68 | + System.err.println("Downloading: " + sourceUrl); |
| 69 | + System.err.println(" to: " + dest.toAbsolutePath()); |
| 70 | + |
| 71 | + URL url = new URL(sourceUrl); |
| 72 | + HttpURLConnection conn = openConnection(url); |
| 73 | + long total = conn.getContentLengthLong(); |
| 74 | + long bytesRead = 0L; |
| 75 | + long lastReport = 0L; |
| 76 | + byte[] buf = new byte[65_536]; |
| 77 | + |
| 78 | + try (InputStream in = conn.getInputStream(); |
| 79 | + OutputStream out = new BufferedOutputStream(new FileOutputStream(dest.toFile()))) { |
| 80 | + int n; |
| 81 | + while ((n = in.read(buf)) > 0) { |
| 82 | + out.write(buf, 0, n); |
| 83 | + bytesRead += n; |
| 84 | + if (bytesRead - lastReport >= 50L << 20) { |
| 85 | + lastReport = bytesRead; |
| 86 | + String progress = total > 0 |
| 87 | + ? String.format("%.0f%%", 100.0 * bytesRead / total) |
| 88 | + : String.format("%.0f MB received", bytesRead / 1e6); |
| 89 | + System.err.printf(" %.1f MB [%s]%n", bytesRead / 1e6, progress); |
| 90 | + } |
| 91 | + } |
| 92 | + } finally { |
| 93 | + conn.disconnect(); |
| 94 | + } |
| 95 | + System.err.printf(" Download complete: %.1f MB%n", bytesRead / 1e6); |
| 96 | + } |
| 97 | + |
| 98 | + /** |
| 99 | + * Parses the bulk-data index JSON and returns the {@code download_uri} for |
| 100 | + * the entry whose {@code type} matches {@code targetType}. |
| 101 | + */ |
| 102 | + static String parseDownloadUri(String json, String targetType) throws IOException { |
| 103 | + try (JsonReader reader = new JsonReader(new StringReader(json))) { |
| 104 | + reader.beginObject(); |
| 105 | + while (reader.hasNext()) { |
| 106 | + if ("data".equals(reader.nextName())) { |
| 107 | + reader.beginArray(); |
| 108 | + while (reader.hasNext()) { |
| 109 | + String uri = readEntry(reader, targetType); |
| 110 | + if (uri != null) { |
| 111 | + return uri; |
| 112 | + } |
| 113 | + } |
| 114 | + reader.endArray(); |
| 115 | + } else { |
| 116 | + reader.skipValue(); |
| 117 | + } |
| 118 | + } |
| 119 | + reader.endObject(); |
| 120 | + } catch (IOException e) { |
| 121 | + return null; |
| 122 | + } |
| 123 | + return null; |
| 124 | + } |
| 125 | + |
| 126 | + private static String readEntry(JsonReader reader, String targetType) throws IOException { |
| 127 | + String type = null; |
| 128 | + String downloadUri = null; |
| 129 | + reader.beginObject(); |
| 130 | + while (reader.hasNext()) { |
| 131 | + String name = reader.nextName(); |
| 132 | + if ("type".equals(name)) { |
| 133 | + type = reader.nextString(); |
| 134 | + } else if ("download_uri".equals(name)) { |
| 135 | + downloadUri = reader.nextString(); |
| 136 | + } else { |
| 137 | + reader.skipValue(); |
| 138 | + } |
| 139 | + } |
| 140 | + reader.endObject(); |
| 141 | + return targetType.equals(type) ? downloadUri : null; |
| 142 | + } |
| 143 | + |
| 144 | + private static String fetchText(String urlStr) throws IOException { |
| 145 | + URL url = new URL(urlStr); |
| 146 | + HttpURLConnection conn = openConnection(url); |
| 147 | + try { |
| 148 | + byte[] data = conn.getInputStream().readAllBytes(); |
| 149 | + return new String(data, StandardCharsets.UTF_8); |
| 150 | + } finally { |
| 151 | + conn.disconnect(); |
| 152 | + } |
| 153 | + } |
| 154 | + |
| 155 | + private static HttpURLConnection openConnection(URL url) throws IOException { |
| 156 | + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); |
| 157 | + conn.setRequestProperty("User-Agent", "forge-scryfall-uuid-map/1.0"); |
| 158 | + conn.setConnectTimeout(CONNECT_TIMEOUT); |
| 159 | + conn.setReadTimeout(READ_TIMEOUT); |
| 160 | + conn.setInstanceFollowRedirects(true); |
| 161 | + conn.connect(); |
| 162 | + int code = conn.getResponseCode(); |
| 163 | + if (code != HttpURLConnection.HTTP_OK) { |
| 164 | + conn.disconnect(); |
| 165 | + throw new IOException("HTTP " + code + " fetching " + url); |
| 166 | + } |
| 167 | + return conn; |
| 168 | + } |
| 169 | +} |
0 commit comments