Add Wikipedia fallback so every game has cover art
libretro-thumbnails stops at the retro consoles, leaving the ten Xbox 360
titles blank. English Wikipedia carries a cover on essentially every
notable game article and needs no account, so it now runs as a second pass
for anything libretro cannot match.
Correcting an earlier claim in this repo: libretro does publish a
"Microsoft - Xbox 360" set. I had reported it as absent after checking only
my own hardcoded system map, not the actual catalogue of 123 sets. The set
turns out to hold about a dozen entries, none of them ours, so the
conclusion held but the reason given was wrong. It is now mapped and
searched anyway, in case it fills out later.
The cover filename is read from the article's infobox rather than inferred
from file names: filtering names for "box" also matches
"Xbox-360-Pro-wController.png". Two details that cost a round each:
* the infobox writes the field both bare ("Halo 3 final boxshot.JPG") and
prefixed ("File:Lost-Planet-New.jpg"), so any prefix is stripped before
exactly one is added back
* the API returns 429 under an unthrottled loop, so calls are spaced one
second apart, retried with a longer backoff, and cached to disk
Coverage is now 105/105 — 93 from libretro, 12 from Wikipedia.
Two rows took art of the right game but the wrong platform, because their
system field looks wrong in the source data: a Game Boy "Donkey Kong
Country 2" and a DS "Donkey Kong Country Returns". Noted in the README
rather than silently corrected.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
+181
-31
@@ -49,6 +49,10 @@ SYSTEM_DIRS = {
|
||||
"PS1": ["Sony - PlayStation"],
|
||||
"PS2": ["Sony - PlayStation 2"],
|
||||
"PSP": ["Sony - PlayStation Portable"],
|
||||
# libretro does publish a "Microsoft - Xbox 360" set, but it is a stub of
|
||||
# about a dozen entries. It is searched anyway in case it fills out later;
|
||||
# in practice these titles fall through to the Wikipedia pass.
|
||||
"360": ["Microsoft - Xbox 360"],
|
||||
}
|
||||
|
||||
# Preferred release region, best first. The library is a US collection, so a USA
|
||||
@@ -138,12 +142,17 @@ def http(url: str, *, data=None, headers=None, method=None, timeout=90) -> bytes
|
||||
request.add_header(key, value)
|
||||
last_error: Exception | None = None
|
||||
|
||||
for attempt in range(3):
|
||||
for attempt in range(4):
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
return response.read()
|
||||
except urllib.error.HTTPError as exc:
|
||||
# 4xx will not improve on retry; surface it immediately.
|
||||
# 429 means we asked too fast, so back off hard and try again.
|
||||
if exc.code == 429:
|
||||
last_error = exc
|
||||
time.sleep(5 * (attempt + 1))
|
||||
continue
|
||||
# Other 4xx will not improve on retry; surface immediately.
|
||||
if exc.code < 500:
|
||||
raise
|
||||
last_error = exc
|
||||
@@ -151,7 +160,7 @@ def http(url: str, *, data=None, headers=None, method=None, timeout=90) -> bytes
|
||||
last_error = exc
|
||||
time.sleep(1.5 * (attempt + 1))
|
||||
|
||||
raise RuntimeError(f"GET {url} failed after 3 attempts: {last_error}")
|
||||
raise RuntimeError(f"GET {url} failed after 4 attempts: {last_error}")
|
||||
|
||||
|
||||
def api_json(base: str, path: str, token: str | None = None, *, data=None, method=None):
|
||||
@@ -187,6 +196,112 @@ def upload_image(base: str, token: str, filename: str, blob: bytes) -> dict:
|
||||
return json.loads(raw)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Wikipedia fallback
|
||||
# --------------------------------------------------------------------------
|
||||
#
|
||||
# libretro's coverage stops at the retro consoles — its "Microsoft - Xbox 360"
|
||||
# set exists but holds only a dozen entries, none of them ours. English
|
||||
# Wikipedia carries a cover image on essentially every notable game article and
|
||||
# needs no account.
|
||||
#
|
||||
# The image is read from the article's infobox rather than guessed from file
|
||||
# names: filtering names for "box" also matches "Xbox-360-Pro-wController.png".
|
||||
|
||||
WIKI_API = "https://en.wikipedia.org/w/api.php"
|
||||
WIKI_UA = "LudosData/1.0 (personal game library; https://github.com/)"
|
||||
WIKI_MIN_INTERVAL = 1.0 # seconds between calls; the API 429s if pushed
|
||||
|
||||
_wiki_last_call = 0.0
|
||||
|
||||
|
||||
def wiki_api(**params):
|
||||
"""Throttled Wikipedia API call."""
|
||||
global _wiki_last_call
|
||||
|
||||
wait = WIKI_MIN_INTERVAL - (time.monotonic() - _wiki_last_call)
|
||||
if wait > 0:
|
||||
time.sleep(wait)
|
||||
|
||||
params.setdefault("action", "query")
|
||||
params.setdefault("format", "json")
|
||||
raw = http(f"{WIKI_API}?{urllib.parse.urlencode(params)}", headers={"User-Agent": WIKI_UA})
|
||||
_wiki_last_call = time.monotonic()
|
||||
return json.loads(raw)
|
||||
|
||||
|
||||
def wiki_article(title: str, cache_dir: Path) -> tuple[str, str] | None:
|
||||
"""(resolved title, wikitext) for a page, or None when it does not exist."""
|
||||
safe = re.sub(r"[^A-Za-z0-9._ -]", "_", title)[:120]
|
||||
cache = cache_dir / "wiki" / f"{safe}.json"
|
||||
|
||||
if cache.exists():
|
||||
blob = json.loads(cache.read_text())
|
||||
return (blob["title"], blob["text"]) if blob else None
|
||||
|
||||
data = wiki_api(prop="revisions", rvprop="content", rvslots="main",
|
||||
titles=title, redirects=1)
|
||||
page = next(iter(data["query"]["pages"].values()))
|
||||
|
||||
result = None
|
||||
if "missing" not in page and page.get("revisions"):
|
||||
result = (page["title"], page["revisions"][0]["slots"]["main"]["*"])
|
||||
|
||||
cache.parent.mkdir(parents=True, exist_ok=True)
|
||||
cache.write_text(json.dumps({"title": result[0], "text": result[1]} if result else None))
|
||||
return result
|
||||
|
||||
|
||||
def wiki_cover_url(game: dict, cache_dir: str | Path) -> tuple[str, str] | None:
|
||||
"""(image URL, source description) for a game's cover, or None."""
|
||||
cache_dir = Path(cache_dir)
|
||||
title, year, system = game["title"], game.get("year"), game.get("system")
|
||||
|
||||
# Bare title first, then the disambiguated forms Wikipedia actually uses.
|
||||
attempts = [title, f"{title} (video game)"]
|
||||
if year:
|
||||
attempts.append(f"{title} ({year} video game)")
|
||||
|
||||
article = None
|
||||
for candidate in attempts:
|
||||
found = wiki_article(candidate, cache_dir)
|
||||
if found and "Infobox video game" in found[1]:
|
||||
article = found
|
||||
break
|
||||
|
||||
if not article:
|
||||
return None
|
||||
|
||||
page_title, text = article
|
||||
|
||||
# Guard against landing on the film/book of the same name, or a different
|
||||
# entry in the series: the article should mention the platform we hold.
|
||||
if system == "360" and "Xbox 360" not in text:
|
||||
return None
|
||||
|
||||
match = re.search(r"\|\s*image\s*=\s*([^\n|]+)", text)
|
||||
if not match:
|
||||
return None
|
||||
|
||||
filename = match.group(1).strip()
|
||||
filename = re.sub(r"<!--.*?-->", "", filename).strip()
|
||||
filename = filename.strip("[]| ")
|
||||
# The field is written both ways: bare ("Halo 3 final boxshot.JPG") and
|
||||
# prefixed ("File:Lost-Planet-New.jpg"), with or without [[ ]] around it.
|
||||
# Strip any prefix so exactly one is added back below.
|
||||
filename = re.sub(r"^\s*(?:File|Image)\s*:\s*", "", filename, flags=re.I).strip()
|
||||
if not filename:
|
||||
return None
|
||||
|
||||
info = wiki_api(prop="imageinfo", titles=f"File:{filename}",
|
||||
iiprop="url|mime", iiurlwidth=600)
|
||||
page = next(iter(info["query"]["pages"].values()))
|
||||
image_info = (page.get("imageinfo") or [{}])[0]
|
||||
url = image_info.get("thumburl") or image_info.get("url")
|
||||
|
||||
return (url, f"wikipedia:{page_title}") if url else None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Listings
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -252,6 +367,7 @@ def best_match(title: str, candidates: list[Candidate]) -> tuple[Candidate | Non
|
||||
# Capped below 0.95 so a genuine exact title always outranks it, and
|
||||
# scaled by coverage so the closest-length candidate wins among several
|
||||
# ("Donkey Kong Country 3" beats a bare "Donkey Kong Country").
|
||||
#
|
||||
# Guard against collapsing a sequel onto its base game. If we are asking
|
||||
# for a number the candidate does not have — "Donkey Kong Country 2"
|
||||
# against a plain "Donkey Kong Country" — containment would happily match
|
||||
@@ -294,6 +410,8 @@ def main() -> int:
|
||||
parser.add_argument("--overwrite", action="store_true", help="replace art that is already set")
|
||||
parser.add_argument("--min-score", type=float, default=0.80,
|
||||
help="similarity below which a match is not applied (0-1)")
|
||||
parser.add_argument("--no-wikipedia", action="store_true",
|
||||
help="skip the Wikipedia fallback; use libretro only")
|
||||
parser.add_argument("--cache", default=str(Path(__file__).parent / ".cache"))
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -326,8 +444,18 @@ def main() -> int:
|
||||
print()
|
||||
|
||||
applied = skipped = failed = 0
|
||||
unsupported: list[dict] = []
|
||||
from_wiki = 0
|
||||
weak: list[tuple[dict, str, float]] = []
|
||||
unmatched: list[dict] = []
|
||||
|
||||
def attach(game: dict, blob: bytes, filename: str) -> None:
|
||||
"""Upload the image and point the game at it."""
|
||||
uploaded = upload_image(base, token, filename, blob)
|
||||
payload = {k: game.get(k) for k in (
|
||||
"title", "system", "genre", "year", "developer", "publisher",
|
||||
"description", "own", "dumped", "played", "finished")}
|
||||
payload["art"] = uploaded["fileName"]
|
||||
api_json(base, f"/api/games/{game['id']}", token, data=payload, method="PUT")
|
||||
|
||||
for game in sorted(games, key=lambda g: g["title"].lower()):
|
||||
title, system = game["title"], game["system"]
|
||||
@@ -336,58 +464,80 @@ def main() -> int:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
if not system or system not in SYSTEM_DIRS:
|
||||
unsupported.append(game)
|
||||
# --- pass 1: libretro, which has the better art where it has any ----
|
||||
candidate, score = (None, 0.0)
|
||||
if system in SYSTEM_DIRS:
|
||||
candidate, score = best_match(title, listings.get(system, []))
|
||||
|
||||
if candidate and score >= args.min_score:
|
||||
flag = " " if score >= 0.95 else "~"
|
||||
print(f" {flag} {system:4} {title[:46]:48} {score:.2f} {candidate.filename[:50]}")
|
||||
if score < 0.95:
|
||||
weak.append((game, candidate.filename, score))
|
||||
|
||||
if args.dry_run:
|
||||
applied += 1
|
||||
continue
|
||||
try:
|
||||
directory = urllib.parse.quote(candidate.directory)
|
||||
name = urllib.parse.quote(candidate.filename)
|
||||
attach(game, http(f"{THUMBNAIL_HOST}/{directory}/Named_Boxarts/{name}"),
|
||||
candidate.filename)
|
||||
applied += 1
|
||||
except Exception as exc: # noqa: BLE001 - report and continue
|
||||
print(f" -> FAILED: {exc}")
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
candidate, score = best_match(title, listings[system])
|
||||
if not candidate or score < args.min_score:
|
||||
# --- pass 2: Wikipedia, for anything libretro does not carry --------
|
||||
if args.no_wikipedia:
|
||||
print(f" ? {system:4} {title[:46]:48} no match")
|
||||
unmatched.append(game)
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
flag = " " if score >= 0.95 else "~"
|
||||
print(f" {flag} {system:4} {title[:46]:48} {score:.2f} {candidate.filename[:52]}")
|
||||
if score < 0.95:
|
||||
weak.append((game, candidate.filename, score))
|
||||
try:
|
||||
found = wiki_cover_url(game, cache_dir)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f" ! {system:4} {title[:46]:48} wikipedia error: {exc}")
|
||||
found = None
|
||||
|
||||
if not found:
|
||||
print(f" ? {system:4} {title[:46]:48} no match")
|
||||
unmatched.append(game)
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
url, source = found
|
||||
print(f" W {system:4} {title[:46]:48} {source[:50]}")
|
||||
|
||||
if args.dry_run:
|
||||
applied += 1
|
||||
from_wiki += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
directory = urllib.parse.quote(candidate.directory)
|
||||
name = urllib.parse.quote(candidate.filename)
|
||||
blob = http(f"{THUMBNAIL_HOST}/{directory}/Named_Boxarts/{name}")
|
||||
|
||||
uploaded = upload_image(base, token, candidate.filename, blob)
|
||||
|
||||
payload = {k: game.get(k) for k in (
|
||||
"title", "system", "genre", "year", "developer", "publisher",
|
||||
"description", "own", "dumped", "played", "finished")}
|
||||
payload["art"] = uploaded["fileName"]
|
||||
api_json(base, f"/api/games/{game['id']}", token, data=payload, method="PUT")
|
||||
attach(game, http(url, headers={"User-Agent": WIKI_UA}), url.split("/")[-1])
|
||||
applied += 1
|
||||
except Exception as exc: # noqa: BLE001 - report and continue the batch
|
||||
from_wiki += 1
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f" -> FAILED: {exc}")
|
||||
failed += 1
|
||||
|
||||
# ---- report ----------------------------------------------------------
|
||||
print("\n" + "=" * 72)
|
||||
verb = "would attach" if args.dry_run else "attached"
|
||||
print(f"{verb}: {applied} already had art: {skipped} no match: {failed} "
|
||||
f"unsupported system: {len(unsupported)}")
|
||||
print(f"{verb}: {applied} (libretro {applied - from_wiki}, wikipedia {from_wiki}) "
|
||||
f"already had art: {skipped} still missing: {failed}")
|
||||
|
||||
if weak:
|
||||
print(f"\nWorth eyeballing in the UI — matched below 0.95 similarity ({len(weak)}):")
|
||||
for game, filename, score in sorted(weak, key=lambda w: w[2]):
|
||||
print(f" {score:.2f} {game['system']:4} {game['title'][:40]:42} -> {filename[:50]}")
|
||||
|
||||
if unsupported:
|
||||
systems_missing = sorted({g['system'] or '(none)' for g in unsupported})
|
||||
print(f"\nNo libretro thumbnail set for: {', '.join(systems_missing)} "
|
||||
f"({len(unsupported)} games). These need IGDB:")
|
||||
for game in unsupported[:15]:
|
||||
if unmatched:
|
||||
print(f"\nNo cover found from either source ({len(unmatched)}):")
|
||||
for game in unmatched:
|
||||
print(f" {game['system'] or '-':4} {game['title']}")
|
||||
|
||||
return 0
|
||||
|
||||
Reference in New Issue
Block a user