Add archive exporter for the personal BeatLeader mirror

scripts/archive is a CLI that exports BeatLeader and ScoreSaber scores, replays, score stats, leaderboard snapshots and BeatSaver map zips into an append-only archive. Anchor the .gitignore 'archive' rule to the repo root so it no longer hides scripts/archive. Record phase 1 results in the plan. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-08 18:30:02 -08:00
parent 257df6f1d8
commit a759308711
10 changed files with 1093 additions and 32 deletions
@@ -0,0 +1,101 @@
+// ScoreSaber exporter: player snapshot, scores (score + leaderboard items), replays.
+// Public API, no auth. Replays are paced at ~10.5 s (360/hour limit) and run as their own command.
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { SS_SCRAPER_REPLAYS } from './config.ts';
+import { get, limiters } from './http.ts';
+import { archivePath, eachWithProgress, exists, listFiles, readJson, save, saveSnapshotIfChanged, updateState } from './store.ts';
+
+const SS_API = 'https://scoresaber.com/api';
+const ss = (playerId: string, ...p: string[]) => archivePath('scoresaber', playerId, ...p);
+
+export type SSItem = {
+	score: { id: number; timeSet: string; hasReplay: boolean };
+	leaderboard: { songHash: string };
+};
+
+export function archivedSSScores(playerId: string, sinceUnix = 0): SSItem[] {
+	const dir = ss(playerId, 'scores');
+	return listFiles(dir, '.json')
+		.map((f) => readJson<SSItem>(path.join(dir, f)))
+		.filter((it) => Date.parse(it.score.timeSet) / 1000 >= sinceUnix);
+}
+
+export async function exportSSPlayer(playerId: string): Promise<void> {
+	const r = await get(`${SS_API}/player/${playerId}/full`, { limiter: limiters.scoresaber });
+	console.log(`ScoreSaber player ${playerId}: ${saveSnapshotIfChanged(ss(playerId, 'player'), r.body) ? 'saved' : 'unchanged'}`);
+}
+
+/** Page `sort=recent` until scores are older than `sinceUnix`. */
+export async function exportSSScores(playerId: string, sinceUnix: number): Promise<void> {
+	console.log(`ScoreSaber scores for ${playerId} since ${new Date(sinceUnix * 1000).toISOString()}`);
+	let added = 0;
+	let seen = 0;
+	for (let page = 1; ; page++) {
+		const r = await get(`${SS_API}/player/${playerId}/scores?limit=100&page=${page}&sort=recent`, { limiter: limiters.scoresaber });
+		const json = JSON.parse(r.body.toString('utf-8')) as { playerScores: SSItem[]; metadata: { total: number } };
+		let reachedCutoff = false;
+		for (const item of json.playerScores) {
+			if (Date.parse(item.score.timeSet) / 1000 < sinceUnix) {
+				reachedCutoff = true;
+				break;
+			}
+			seen++;
+			const file = ss(playerId, 'scores', `${item.score.id}.json`);
+			if (exists(file)) continue;
+			save(file, JSON.stringify(item));
+			added++;
+		}
+		console.log(`  page ${page}: ${seen} in window, ${added} new (total on server ${json.metadata.total})`);
+		if (reachedCutoff || json.playerScores.length < 100) break;
+	}
+	updateState(`scoresaber:${playerId}`, { scoresSince: sinceUnix, lastScoresRun: new Date().toISOString() });
+}
+
+/** Download replays for archived scores with hasReplay, oldest first, ~10.5 s apart. */
+export async function exportSSReplays(playerId: string, sinceUnix: number): Promise<void> {
+	const items = archivedSSScores(playerId, sinceUnix)
+		.filter((it) => it.score.hasReplay)
+		.sort((a, b) => Date.parse(a.score.timeSet) - Date.parse(b.score.timeSet));
+	seedFromScraper(playerId, items);
+	const todo = items.filter((it) => !exists(ss(playerId, 'replays', `${it.score.id}.dat`)));
+	console.log(`ScoreSaber replays for ${playerId}: ${items.length} with hasReplay, ${todo.length} to download (~${Math.ceil((todo.length * 10.5) / 60)} min)`);
+	const res = await eachWithProgress(
+		'ss-replays',
+		todo,
+		async (it) => {
+			const r = await get(`${SS_API}/v2/scores/${it.score.id}/replay`, {
+				limiter: limiters.scoresaberReplay,
+				accept: [404],
+				timeoutMs: 300_000
+			});
+			if (r.status === 404) return 'missing';
+			save(ss(playerId, 'replays', `${it.score.id}.dat`), r.body);
+			return 'done';
+		},
+		10
+	);
+	if (res.missing || res.failed.length) {
+		updateState('failures', { [`scoresaber:${playerId}:replays`]: res.failed.map((f) => f.item.score.id) });
+	}
+}
+
+/** Copy replays ss-scraper already downloaded (<scoreId>_<name>.dat) instead of fetching them again. */
+function seedFromScraper(playerId: string, items: SSItem[]): void {
+	const dir = path.join(SS_SCRAPER_REPLAYS, playerId);
+	const byId = new Map<string, string>();
+	for (const f of listFiles(dir, '.dat')) byId.set(f.split('_')[0], f);
+	let copied = 0;
+	for (const it of items) {
+		const src = byId.get(String(it.score.id));
+		const dest = ss(playerId, 'replays', `${it.score.id}.dat`);
+		if (!src || exists(dest)) continue;
+		save(dest, fs.readFileSync(path.join(dir, src)));
+		copied++;
+	}
+	if (copied) console.log(`  copied ${copied} replays from ${dir}`);
+}
+
+export function ssHashes(playerId: string, sinceUnix = 0): string[] {
+	return [...new Set(archivedSSScores(playerId, sinceUnix).map((it) => it.leaderboard.songHash.toLowerCase()))].sort();
+}