Add archive exporter for the personal BeatLeader mirror

scripts/archive is a CLI that exports BeatLeader and ScoreSaber scores, replays, score stats, leaderboard snapshots and BeatSaver map zips into an append-only archive. Anchor the .gitignore 'archive' rule to the repo root so it no longer hides scripts/archive. Record phase 1 results in the plan. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-08 18:30:02 -08:00
parent 257df6f1d8
commit a759308711
10 changed files with 1093 additions and 32 deletions
@@ -0,0 +1,89 @@
+// BeatSaver exporter: map JSON and the zip of the exact version hash that was played.
+import * as fs from 'node:fs';
+import { get, limiters } from './http.ts';
+import { archivePath, eachWithProgress, exists, readJson, save } from './store.ts';
+
+const bs = (...p: string[]) => archivePath('beatsaver', ...p);
+const MISSING_FILE = () => bs('missing.json');
+
+type BSMap = { id: string; versions: { hash: string; downloadURL: string }[] };
+type Missing = Record<string, { reason: string; checkedAt: string; recoveredFrom?: string }>;
+
+function readMissing(): Missing {
+	try {
+		return readJson<Missing>(MISSING_FILE());
+	} catch {
+		return {};
+	}
+}
+
+/** Entry names from a zip's central directory, or null if the zip is truncated or malformed. */
+export function zipEntryNames(buf: Buffer): string[] | null {
+	if (buf.length < 22 || buf.readUInt32LE(0) !== 0x04034b50) return null;
+	const tailStart = Math.max(0, buf.length - 65_557);
+	const eocd = buf.subarray(tailStart).lastIndexOf(Buffer.from([0x50, 0x4b, 0x05, 0x06]));
+	if (eocd < 0) return null;
+	const e = tailStart + eocd;
+	const count = buf.readUInt16LE(e + 10);
+	let p = buf.readUInt32LE(e + 16);
+	const names: string[] = [];
+	for (let i = 0; i < count; i++) {
+		if (p + 46 > buf.length || buf.readUInt32LE(p) !== 0x02014b50) return null;
+		const nameLen = buf.readUInt16LE(p + 28);
+		const extraLen = buf.readUInt16LE(p + 30);
+		const commentLen = buf.readUInt16LE(p + 32);
+		names.push(buf.toString('utf-8', p + 46, p + 46 + nameLen));
+		p += 46 + nameLen + extraLen + commentLen;
+	}
+	return names;
+}
+
+/** A zip is usable if its central directory is intact and lists an Info.dat (any case, any folder). */
+export function looksLikeMapZip(buf: Buffer): boolean {
+	return zipEntryNames(buf)?.some((n) => /(^|\/)info\.dat$/i.test(n)) ?? false;
+}
+
+export async function exportMaps(hashes: string[]): Promise<void> {
+	const missing = readMissing();
+	console.log(`BeatSaver maps: ${hashes.length} hashes`);
+	const res = await eachWithProgress('maps', hashes, async (hash) => {
+		const jsonFile = bs('maps', `${hash}.json`);
+		const zipFile = bs('zips', `${hash}.zip`);
+		if (exists(zipFile) && exists(jsonFile)) return 'skipped';
+
+		let map: BSMap | null = null;
+		if (exists(jsonFile)) {
+			map = readJson<BSMap>(jsonFile);
+		} else {
+			const r = await get(`https://api.beatsaver.com/maps/hash/${hash}`, { limiter: limiters.beatsaver, accept: [404] });
+			if (r.status === 200) {
+				save(jsonFile, r.body);
+				map = JSON.parse(r.body.toString('utf-8')) as BSMap;
+			}
+		}
+		if (exists(zipFile)) return 'done';
+
+		// Prefer the version entry for this exact hash; old versions often remain on the CDN.
+		const version = map?.versions.find((v) => v.hash.toLowerCase() === hash);
+		const candidates = [version?.downloadURL, `https://r2cdn.beatsaver.com/${hash}.zip`, `https://cdn.beatsaver.com/${hash}.zip`].filter(
+			(u): u is string => Boolean(u)
+		);
+		for (const url of [...new Set(candidates)]) {
+			const r = await get(url, { limiter: limiters.beatsaverCdn, accept: [403, 404] });
+			if (r.status === 200 && looksLikeMapZip(r.body)) {
+				save(zipFile, r.body);
+				delete missing[hash];
+				return 'done';
+			}
+		}
+		missing[hash] = {
+			reason: map ? `map ${map.id} found but version zip unavailable` : 'hash unknown to BeatSaver',
+			checkedAt: new Date().toISOString()
+		};
+		return 'missing';
+	});
+	fs.mkdirSync(bs(), { recursive: true });
+	save(MISSING_FILE(), JSON.stringify(missing, null, 2) + '\n');
+	console.log(`  missing.json: ${Object.keys(missing).length} hashes`);
+	if (res.failed.length) console.warn(`  ${res.failed.length} failed; rerun to retry`);
+}