diff --git a/README.md b/README.md index a694b2e..6453a85 100644 --- a/README.md +++ b/README.md @@ -77,21 +77,33 @@ cd backend && dotnet build # 0 warnings expected ### Cover art -`tools/cover-art/fetch_art.py` fills in box art from -[libretro-thumbnails](https://thumbnails.libretro.com), matching on title + -system and pushing each image through the app's own `POST /api/images`, so it -gets the same validation and WebP re-encoding as a manual upload. Standard -library only — no virtualenv needed. +`tools/cover-art/fetch_art.py` fills in box art, pushing each image through the +app's own `POST /api/images` so it gets the same validation and WebP re-encoding +as a manual upload. Standard library only — no virtualenv, and neither source +needs an account. + +It tries two sources in order: + +1. **[libretro-thumbnails](https://thumbnails.libretro.com)** — scanned retail + boxes, named to the No-Intro / Redump conventions. Best art where it has any, + but its coverage is the retro consoles. Its `Microsoft - Xbox 360` set exists + but holds about a dozen entries. +2. **English Wikipedia** — a cover on essentially every notable game article, + which is what fills the Xbox 360 shelf. The exact filename is read from the + article's infobox rather than guessed from file names, since filtering names + for "box" also matches `Xbox-360-Pro-wController.png`. Calls are throttled to + one per second and cached; the API returns 429 if pushed harder. ```bash cd tools/cover-art -python3 fetch_art.py --password '...' --dry-run # report matches, change nothing -python3 fetch_art.py --password '...' # download and attach -python3 fetch_art.py --password '...' --overwrite # also replace existing art +python3 fetch_art.py --password '...' --dry-run # report matches, change nothing +python3 fetch_art.py --password '...' # download and attach +python3 fetch_art.py --password '...' --overwrite # also replace existing art +python3 fetch_art.py --password '...' --no-wikipedia # libretro only ``` -Always dry-run first; it prints every match with a similarity score and flags -anything below 0.95 for eyeballing. +Always dry-run first; it prints every match with a similarity score, marks the +source (`W` for Wikipedia), and flags anything below 0.95 for eyeballing. Matching handles the gaps between a personal catalogue and a ROM-naming one: accents (`Pokemon` → `Pokémon`), roman numerals (our SNES `Final Fantasy 2` is @@ -100,9 +112,13 @@ subtitles in either direction, and outright typos — `Brett Hull Hocky 95` find `Brett Hull Hockey 95`. A sequel guard stops `Donkey Kong Country 2` from silently taking `Donkey Kong Country`'s box. -**Xbox 360 is not covered** — libretro has no thumbnail set for it, so those 10 -titles are reported as unsupported. They need a source such as IGDB, which -requires a free Twitch developer client ID and secret. +All 105 games currently have art: 93 from libretro, 12 from Wikipedia. + +Two rows got art of the right *game* but the wrong *platform*, because the +platform in the source data looks wrong — a Game Boy "Donkey Kong Country 2" +(never released on that system; the handheld sequels were *Donkey Kong Land*) +and a DS "Donkey Kong Country Returns" (a Wii game, later *Returns 3D* on 3DS). +Fix the system field and re-run with `--overwrite` to correct them. Art is publisher copyright. Fetching it for a private collection is ordinary practice for library software; redistributing it is a different question. diff --git a/tools/cover-art/fetch_art.py b/tools/cover-art/fetch_art.py index f49917f..5703524 100644 --- a/tools/cover-art/fetch_art.py +++ b/tools/cover-art/fetch_art.py @@ -49,6 +49,10 @@ SYSTEM_DIRS = { "PS1": ["Sony - PlayStation"], "PS2": ["Sony - PlayStation 2"], "PSP": ["Sony - PlayStation Portable"], + # libretro does publish a "Microsoft - Xbox 360" set, but it is a stub of + # about a dozen entries. It is searched anyway in case it fills out later; + # in practice these titles fall through to the Wikipedia pass. + "360": ["Microsoft - Xbox 360"], } # Preferred release region, best first. The library is a US collection, so a USA @@ -138,12 +142,17 @@ def http(url: str, *, data=None, headers=None, method=None, timeout=90) -> bytes request.add_header(key, value) last_error: Exception | None = None - for attempt in range(3): + for attempt in range(4): try: with urllib.request.urlopen(request, timeout=timeout) as response: return response.read() except urllib.error.HTTPError as exc: - # 4xx will not improve on retry; surface it immediately. + # 429 means we asked too fast, so back off hard and try again. + if exc.code == 429: + last_error = exc + time.sleep(5 * (attempt + 1)) + continue + # Other 4xx will not improve on retry; surface immediately. if exc.code < 500: raise last_error = exc @@ -151,7 +160,7 @@ def http(url: str, *, data=None, headers=None, method=None, timeout=90) -> bytes last_error = exc time.sleep(1.5 * (attempt + 1)) - raise RuntimeError(f"GET {url} failed after 3 attempts: {last_error}") + raise RuntimeError(f"GET {url} failed after 4 attempts: {last_error}") def api_json(base: str, path: str, token: str | None = None, *, data=None, method=None): @@ -187,6 +196,112 @@ def upload_image(base: str, token: str, filename: str, blob: bytes) -> dict: return json.loads(raw) +# -------------------------------------------------------------------------- +# Wikipedia fallback +# -------------------------------------------------------------------------- +# +# libretro's coverage stops at the retro consoles — its "Microsoft - Xbox 360" +# set exists but holds only a dozen entries, none of them ours. English +# Wikipedia carries a cover image on essentially every notable game article and +# needs no account. +# +# The image is read from the article's infobox rather than guessed from file +# names: filtering names for "box" also matches "Xbox-360-Pro-wController.png". + +WIKI_API = "https://en.wikipedia.org/w/api.php" +WIKI_UA = "LudosData/1.0 (personal game library; https://github.com/)" +WIKI_MIN_INTERVAL = 1.0 # seconds between calls; the API 429s if pushed + +_wiki_last_call = 0.0 + + +def wiki_api(**params): + """Throttled Wikipedia API call.""" + global _wiki_last_call + + wait = WIKI_MIN_INTERVAL - (time.monotonic() - _wiki_last_call) + if wait > 0: + time.sleep(wait) + + params.setdefault("action", "query") + params.setdefault("format", "json") + raw = http(f"{WIKI_API}?{urllib.parse.urlencode(params)}", headers={"User-Agent": WIKI_UA}) + _wiki_last_call = time.monotonic() + return json.loads(raw) + + +def wiki_article(title: str, cache_dir: Path) -> tuple[str, str] | None: + """(resolved title, wikitext) for a page, or None when it does not exist.""" + safe = re.sub(r"[^A-Za-z0-9._ -]", "_", title)[:120] + cache = cache_dir / "wiki" / f"{safe}.json" + + if cache.exists(): + blob = json.loads(cache.read_text()) + return (blob["title"], blob["text"]) if blob else None + + data = wiki_api(prop="revisions", rvprop="content", rvslots="main", + titles=title, redirects=1) + page = next(iter(data["query"]["pages"].values())) + + result = None + if "missing" not in page and page.get("revisions"): + result = (page["title"], page["revisions"][0]["slots"]["main"]["*"]) + + cache.parent.mkdir(parents=True, exist_ok=True) + cache.write_text(json.dumps({"title": result[0], "text": result[1]} if result else None)) + return result + + +def wiki_cover_url(game: dict, cache_dir: str | Path) -> tuple[str, str] | None: + """(image URL, source description) for a game's cover, or None.""" + cache_dir = Path(cache_dir) + title, year, system = game["title"], game.get("year"), game.get("system") + + # Bare title first, then the disambiguated forms Wikipedia actually uses. + attempts = [title, f"{title} (video game)"] + if year: + attempts.append(f"{title} ({year} video game)") + + article = None + for candidate in attempts: + found = wiki_article(candidate, cache_dir) + if found and "Infobox video game" in found[1]: + article = found + break + + if not article: + return None + + page_title, text = article + + # Guard against landing on the film/book of the same name, or a different + # entry in the series: the article should mention the platform we hold. + if system == "360" and "Xbox 360" not in text: + return None + + match = re.search(r"\|\s*image\s*=\s*([^\n|]+)", text) + if not match: + return None + + filename = match.group(1).strip() + filename = re.sub(r"", "", filename).strip() + filename = filename.strip("[]| ") + # The field is written both ways: bare ("Halo 3 final boxshot.JPG") and + # prefixed ("File:Lost-Planet-New.jpg"), with or without [[ ]] around it. + # Strip any prefix so exactly one is added back below. + filename = re.sub(r"^\s*(?:File|Image)\s*:\s*", "", filename, flags=re.I).strip() + if not filename: + return None + + info = wiki_api(prop="imageinfo", titles=f"File:{filename}", + iiprop="url|mime", iiurlwidth=600) + page = next(iter(info["query"]["pages"].values())) + image_info = (page.get("imageinfo") or [{}])[0] + url = image_info.get("thumburl") or image_info.get("url") + + return (url, f"wikipedia:{page_title}") if url else None + + # -------------------------------------------------------------------------- # Listings # -------------------------------------------------------------------------- @@ -252,6 +367,7 @@ def best_match(title: str, candidates: list[Candidate]) -> tuple[Candidate | Non # Capped below 0.95 so a genuine exact title always outranks it, and # scaled by coverage so the closest-length candidate wins among several # ("Donkey Kong Country 3" beats a bare "Donkey Kong Country"). + # # Guard against collapsing a sequel onto its base game. If we are asking # for a number the candidate does not have — "Donkey Kong Country 2" # against a plain "Donkey Kong Country" — containment would happily match @@ -294,6 +410,8 @@ def main() -> int: parser.add_argument("--overwrite", action="store_true", help="replace art that is already set") parser.add_argument("--min-score", type=float, default=0.80, help="similarity below which a match is not applied (0-1)") + parser.add_argument("--no-wikipedia", action="store_true", + help="skip the Wikipedia fallback; use libretro only") parser.add_argument("--cache", default=str(Path(__file__).parent / ".cache")) args = parser.parse_args() @@ -326,8 +444,18 @@ def main() -> int: print() applied = skipped = failed = 0 - unsupported: list[dict] = [] + from_wiki = 0 weak: list[tuple[dict, str, float]] = [] + unmatched: list[dict] = [] + + def attach(game: dict, blob: bytes, filename: str) -> None: + """Upload the image and point the game at it.""" + uploaded = upload_image(base, token, filename, blob) + payload = {k: game.get(k) for k in ( + "title", "system", "genre", "year", "developer", "publisher", + "description", "own", "dumped", "played", "finished")} + payload["art"] = uploaded["fileName"] + api_json(base, f"/api/games/{game['id']}", token, data=payload, method="PUT") for game in sorted(games, key=lambda g: g["title"].lower()): title, system = game["title"], game["system"] @@ -336,58 +464,80 @@ def main() -> int: skipped += 1 continue - if not system or system not in SYSTEM_DIRS: - unsupported.append(game) + # --- pass 1: libretro, which has the better art where it has any ---- + candidate, score = (None, 0.0) + if system in SYSTEM_DIRS: + candidate, score = best_match(title, listings.get(system, [])) + + if candidate and score >= args.min_score: + flag = " " if score >= 0.95 else "~" + print(f" {flag} {system:4} {title[:46]:48} {score:.2f} {candidate.filename[:50]}") + if score < 0.95: + weak.append((game, candidate.filename, score)) + + if args.dry_run: + applied += 1 + continue + try: + directory = urllib.parse.quote(candidate.directory) + name = urllib.parse.quote(candidate.filename) + attach(game, http(f"{THUMBNAIL_HOST}/{directory}/Named_Boxarts/{name}"), + candidate.filename) + applied += 1 + except Exception as exc: # noqa: BLE001 - report and continue + print(f" -> FAILED: {exc}") + failed += 1 continue - candidate, score = best_match(title, listings[system]) - if not candidate or score < args.min_score: + # --- pass 2: Wikipedia, for anything libretro does not carry -------- + if args.no_wikipedia: print(f" ? {system:4} {title[:46]:48} no match") + unmatched.append(game) failed += 1 continue - flag = " " if score >= 0.95 else "~" - print(f" {flag} {system:4} {title[:46]:48} {score:.2f} {candidate.filename[:52]}") - if score < 0.95: - weak.append((game, candidate.filename, score)) + try: + found = wiki_cover_url(game, cache_dir) + except Exception as exc: # noqa: BLE001 + print(f" ! {system:4} {title[:46]:48} wikipedia error: {exc}") + found = None + + if not found: + print(f" ? {system:4} {title[:46]:48} no match") + unmatched.append(game) + failed += 1 + continue + + url, source = found + print(f" W {system:4} {title[:46]:48} {source[:50]}") if args.dry_run: applied += 1 + from_wiki += 1 continue try: - directory = urllib.parse.quote(candidate.directory) - name = urllib.parse.quote(candidate.filename) - blob = http(f"{THUMBNAIL_HOST}/{directory}/Named_Boxarts/{name}") - - uploaded = upload_image(base, token, candidate.filename, blob) - - payload = {k: game.get(k) for k in ( - "title", "system", "genre", "year", "developer", "publisher", - "description", "own", "dumped", "played", "finished")} - payload["art"] = uploaded["fileName"] - api_json(base, f"/api/games/{game['id']}", token, data=payload, method="PUT") + attach(game, http(url, headers={"User-Agent": WIKI_UA}), url.split("/")[-1]) applied += 1 - except Exception as exc: # noqa: BLE001 - report and continue the batch + from_wiki += 1 + except Exception as exc: # noqa: BLE001 print(f" -> FAILED: {exc}") failed += 1 # ---- report ---------------------------------------------------------- print("\n" + "=" * 72) verb = "would attach" if args.dry_run else "attached" - print(f"{verb}: {applied} already had art: {skipped} no match: {failed} " - f"unsupported system: {len(unsupported)}") + print(f"{verb}: {applied} (libretro {applied - from_wiki}, wikipedia {from_wiki}) " + f"already had art: {skipped} still missing: {failed}") if weak: print(f"\nWorth eyeballing in the UI — matched below 0.95 similarity ({len(weak)}):") for game, filename, score in sorted(weak, key=lambda w: w[2]): print(f" {score:.2f} {game['system']:4} {game['title'][:40]:42} -> {filename[:50]}") - if unsupported: - systems_missing = sorted({g['system'] or '(none)' for g in unsupported}) - print(f"\nNo libretro thumbnail set for: {', '.join(systems_missing)} " - f"({len(unsupported)} games). These need IGDB:") - for game in unsupported[:15]: + if unmatched: + print(f"\nNo cover found from either source ({len(unmatched)}):") + for game in unmatched: print(f" {game['system'] or '-':4} {game['title']}") return 0