Add archive exporter for the personal BeatLeader mirror
scripts/archive is a CLI that exports BeatLeader and ScoreSaber scores, replays, score stats, leaderboard snapshots and BeatSaver map zips into an append-only archive. Anchor the .gitignore 'archive' rule to the repo root so it no longer hides scripts/archive. Record phase 1 results in the plan. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,236 @@
|
||||
#!/usr/bin/env node
|
||||
// plebleader archive CLI. Run from the repo root: node scripts/archive/cli.ts <command> [options]
|
||||
//
|
||||
// auth login | refresh | status the sync's own BeatLeader OAuth token chain
|
||||
// export bl [--since D] BL profile, scores, replays, score stats, leaderboards
|
||||
// export ss [--since D] ScoreSaber player snapshot and scores (fast)
|
||||
// export ss-replays [--since D] ScoreSaber replays, ~10.5 s apart (run in background)
|
||||
// export maps [--maps-since D] BeatSaver JSON + zip for played hashes (default: last 2 years,
|
||||
// plus all legacy ScoreSaber maps)
|
||||
// export all [--since D] bl, ss, maps (not ss-replays)
|
||||
// report [--since D] counts and gaps in the archive
|
||||
//
|
||||
// --since: YYYY-MM-DD, unix seconds, or "all" (default: all). --maps-since defaults to the two-year
|
||||
// window (DEFAULT_MAPS_SINCE). Every command skips files that already exist, so reruns only fill gaps.
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { ARCHIVE_DIR, BL_API, DEFAULT_MAPS_SINCE, LEGACY_PLAYER_ID, MAIN_PLAYER_ID, PLAYER_IDS, TOKENS_FILE, USER_AGENT, loadDotEnv, parseSince } from './config.ts';
|
||||
import { getAccessToken, login, readTokens, refresh } from './auth.ts';
|
||||
import { archivedScores, blHashes, exportLeaderboards, exportProfile, exportReplays, exportScoreStats, exportScores } from './beatleader.ts';
|
||||
import { archivedSSScores, exportSSPlayer, exportSSReplays, exportSSScores, ssHashes } from './scoresaber.ts';
|
||||
import { exportMaps } from './beatsaver.ts';
|
||||
import { archivePath, exists, listFiles, readState } from './store.ts';
|
||||
|
||||
function fmt(ms: number): string {
|
||||
const d = new Date(ms);
|
||||
const left = ms - Date.now();
|
||||
const hours = Math.round(left / 36e5);
|
||||
return `${d.toISOString()} (${left < 0 ? 'expired' : hours >= 48 ? `${Math.round(hours / 24)} d left` : `${hours} h left`})`;
|
||||
}
|
||||
|
||||
async function blGet<T>(path: string, token: string): Promise<T> {
|
||||
const res = await fetch(`${BL_API}${path}`, {
|
||||
headers: { Authorization: `Bearer ${token}`, 'User-Agent': USER_AGENT }
|
||||
});
|
||||
if (!res.ok) throw new Error(`GET ${path}: HTTP ${res.status}`);
|
||||
return (await res.json()) as T;
|
||||
}
|
||||
|
||||
/**
|
||||
* Phase 1 step 1. The bearer token must belong to pleb, and an unranked map's ratings must be
|
||||
* readable through /leaderboards/hash/{hash}.
|
||||
*
|
||||
* Verified 2026-11-08: BeatLeader only honors bearer tokens on /oauth2/identity, clan management
|
||||
* and ReeSabers presets. /user and the showAllRatings gate on score lists read the beatleader.com
|
||||
* cookie session instead, so score lists always come back with unranked ratings nulled. The
|
||||
* per-hash leaderboard lookup returns full ratings (incl. modifiersRating) even anonymously.
|
||||
*/
|
||||
async function verify(): Promise<boolean> {
|
||||
const token = await getAccessToken();
|
||||
const identity = await blGet<{ id?: string; name?: string }>('/oauth2/identity', token);
|
||||
const idOk = identity.id === MAIN_PLAYER_ID;
|
||||
console.log(`GET /oauth2/identity: ${identity.id} (${identity.name}) ${idOk ? 'OK' : `MISMATCH, expected ${MAIN_PLAYER_ID}`}`);
|
||||
|
||||
type ScorePage = { data: { leaderboard: { song: { name: string; hash: string }; difficulty: { difficultyName: string; status: number } } }[] };
|
||||
// status 3 = ranked; anything else counts as unranked for this check.
|
||||
const page = await blGet<ScorePage>(`/player/${MAIN_PLAYER_ID}/scores?count=50&page=1&sortBy=date&order=desc&type=unranked`, token);
|
||||
const unranked = page.data.find((s) => s.leaderboard.difficulty.status !== 3);
|
||||
if (!unranked) {
|
||||
console.log('No unranked score found in the latest 50 to check ratings against.');
|
||||
return idOk;
|
||||
}
|
||||
type HashLookup = { leaderboards: { difficulty: { difficultyName: string; stars: number | null; techRating: number | null } }[] };
|
||||
const { song, difficulty } = unranked.leaderboard;
|
||||
const lookup = await blGet<HashLookup>(`/leaderboards/hash/${song.hash}`, token);
|
||||
const d = lookup.leaderboards.find((l) => l.difficulty.difficultyName === difficulty.difficultyName)?.difficulty;
|
||||
const ratingsOk = d?.stars != null;
|
||||
console.log(`Unranked "${song.name}" ${difficulty.difficultyName} via /leaderboards/hash: stars=${d?.stars} tech=${d?.techRating} ${ratingsOk ? 'OK' : 'MISSING'}`);
|
||||
return idOk && ratingsOk;
|
||||
}
|
||||
|
||||
function playedHashes(since: number): string[] {
|
||||
return [...new Set([...blHashes(since), ...PLAYER_IDS.flatMap((id) => ssHashes(id, since))])].sort();
|
||||
}
|
||||
|
||||
/**
|
||||
* Hashes to fetch BeatSaver map files for: played within the maps window, plus — regardless of
|
||||
* age — the maps of legacy ScoreSaber scores that have an archived replay, so those replays render.
|
||||
*/
|
||||
function mapHashes(mapsSince: number): string[] {
|
||||
const legacyWithReplay = archivedSSScores(LEGACY_PLAYER_ID, 0)
|
||||
.filter((it) => exists(archivePath('scoresaber', LEGACY_PLAYER_ID, 'replays', `${it.score.id}.dat`)))
|
||||
.map((it) => it.leaderboard.songHash.toLowerCase());
|
||||
return [...new Set([...playedHashes(mapsSince), ...legacyWithReplay])].sort();
|
||||
}
|
||||
|
||||
async function exportBL(since: number): Promise<void> {
|
||||
for (const id of PLAYER_IDS) {
|
||||
await exportProfile(id);
|
||||
await exportScores(id, since);
|
||||
}
|
||||
// The scores dir holds both accounts' scores (keyed by score id).
|
||||
const scores = archivedScores(since);
|
||||
await exportReplays(scores);
|
||||
await exportScoreStats(scores);
|
||||
// BL leaderboard JSON (ratings) for every played hash, not just the maps window: it's small
|
||||
// and the importer needs it for star ratings on old scores.
|
||||
await exportLeaderboards(blHashes(since));
|
||||
}
|
||||
|
||||
async function exportSS(since: number): Promise<void> {
|
||||
for (const id of PLAYER_IDS) {
|
||||
await exportSSPlayer(id);
|
||||
await exportSSScores(id, since);
|
||||
}
|
||||
// Hashes only ScoreSaber knows about still need BL leaderboard data (phase 5 merge).
|
||||
await exportLeaderboards([...new Set(PLAYER_IDS.flatMap((id) => ssHashes(id, since)))].sort());
|
||||
}
|
||||
|
||||
function report(since: number, mapsSince: number): void {
|
||||
const count = (...p: string[]) => listFiles(archivePath(...p)).length;
|
||||
const day = (t: number) => (t ? new Date(t * 1000).toISOString().slice(0, 10) : 'all time');
|
||||
const bl = archivedScores(since);
|
||||
const blWithReplay = bl.filter((s) => s.replay);
|
||||
const blReplayMissing = blWithReplay.filter((s) => !exists(archivePath('beatleader', 'replays', decodeURIComponent(new URL(s.replay!).pathname.split('/').pop()!))));
|
||||
const statsMissing = bl.filter((s) => !exists(archivePath('beatleader', 'scorestats', `${s.id}.json`)));
|
||||
const ss = PLAYER_IDS.map((id) => {
|
||||
const items = archivedSSScores(id, since);
|
||||
const withReplay = items.filter((it) => it.score.hasReplay);
|
||||
const missing = withReplay.filter((it) => !exists(archivePath('scoresaber', id, 'replays', `${it.score.id}.dat`)));
|
||||
return { id, items, withReplay, missing };
|
||||
});
|
||||
const allHashes = playedHashes(since);
|
||||
const noLeaderboard = allHashes.filter((h) => listFiles(archivePath('beatleader', 'leaderboards', h)).length === 0);
|
||||
const hashes = mapHashes(Math.max(since, mapsSince));
|
||||
const noZip = hashes.filter((h) => !exists(archivePath('beatsaver', 'zips', `${h}.zip`)));
|
||||
let missingJson: Record<string, unknown> = {};
|
||||
try {
|
||||
missingJson = JSON.parse(fs.readFileSync(archivePath('beatsaver', 'missing.json'), 'utf-8'));
|
||||
} catch {}
|
||||
const du = (dir: string): number => {
|
||||
let total = 0;
|
||||
const walk = (d: string) => {
|
||||
for (const e of fs.readdirSync(d, { withFileTypes: true })) {
|
||||
const p = path.join(d, e.name);
|
||||
if (e.isDirectory()) walk(p);
|
||||
else total += fs.statSync(p).size;
|
||||
}
|
||||
};
|
||||
if (fs.existsSync(dir)) walk(dir);
|
||||
return total;
|
||||
};
|
||||
|
||||
console.log(`Archive: ${ARCHIVE_DIR}`);
|
||||
console.log(`Window: scores since ${day(since)}, BeatSaver maps since ${day(mapsSince)}`);
|
||||
console.log('');
|
||||
console.log(`BeatLeader scores ${bl.length} in window (${count('beatleader', 'scores')} archived total, both accounts)`);
|
||||
console.log(`BeatLeader replays ${blWithReplay.length - blReplayMissing.length}/${blWithReplay.length}`);
|
||||
console.log(`BeatLeader score stats ${bl.length - statsMissing.length}/${bl.length}`);
|
||||
for (const s of ss) {
|
||||
console.log(`ScoreSaber ${s.id}: ${s.items.length} scores, replays ${s.withReplay.length - s.missing.length}/${s.withReplay.length}`);
|
||||
}
|
||||
console.log(`Played hashes ${allHashes.length}`);
|
||||
console.log(` BL leaderboard snapshot ${allHashes.length - noLeaderboard.length}/${allHashes.length}`);
|
||||
console.log(` BeatSaver zip ${hashes.length - noZip.length}/${hashes.length} in maps window (missing.json lists ${Object.keys(missingJson).length})`);
|
||||
console.log(`Archive size ${(du(ARCHIVE_DIR) / 1e9).toFixed(2)} GB`);
|
||||
const failures = readState().failures;
|
||||
if (failures && Object.keys(failures).length) console.log(`Failures from last runs: ${JSON.stringify(failures)}`);
|
||||
}
|
||||
|
||||
function option(args: string[], name: string): string | undefined {
|
||||
const i = args.indexOf(name);
|
||||
return i >= 0 ? args[i + 1] : args.find((a) => a.startsWith(`${name}=`))?.slice(name.length + 1);
|
||||
}
|
||||
|
||||
async function main(): Promise<number> {
|
||||
loadDotEnv();
|
||||
const args = process.argv.slice(2);
|
||||
const [cmd, sub] = args;
|
||||
|
||||
if (cmd === 'auth' && sub === 'login') {
|
||||
const store = await login();
|
||||
console.log(`\nLogged in as ${store.player_name} (${store.player_id}). Tokens saved to ${TOKENS_FILE}`);
|
||||
console.log(`Refresh token valid until about ${fmt(store.refresh_expires_at)}`);
|
||||
return (await verify()) ? 0 : 1;
|
||||
}
|
||||
if (cmd === 'auth' && sub === 'refresh') {
|
||||
const store = await refresh();
|
||||
console.log(`Refreshed. Access token until ${fmt(store.access_expires_at)}; refresh until about ${fmt(store.refresh_expires_at)}`);
|
||||
return 0;
|
||||
}
|
||||
if (cmd === 'auth' && sub === 'status') {
|
||||
const store = readTokens();
|
||||
if (!store) {
|
||||
console.log(`No tokens at ${TOKENS_FILE}. Run \`auth login\`.`);
|
||||
return 1;
|
||||
}
|
||||
console.log(`Tokens: ${TOKENS_FILE}`);
|
||||
console.log(`Player: ${store.player_name} (${store.player_id})`);
|
||||
console.log(`Access: ${fmt(store.access_expires_at)}`);
|
||||
console.log(`Refresh: ${fmt(store.refresh_expires_at)} (estimated)`);
|
||||
return (await verify()) ? 0 : 1;
|
||||
}
|
||||
|
||||
const since = parseSince(option(args, '--since'));
|
||||
const mapsSince = Math.max(since, parseSince(option(args, '--maps-since'), DEFAULT_MAPS_SINCE));
|
||||
if (cmd === 'export') {
|
||||
console.log(`Archive ${ARCHIVE_DIR}, since ${since ? new Date(since * 1000).toISOString() : 'all time'}\n`);
|
||||
switch (sub) {
|
||||
case 'bl':
|
||||
await exportBL(since);
|
||||
return 0;
|
||||
case 'ss':
|
||||
await exportSS(since);
|
||||
return 0;
|
||||
case 'ss-replays':
|
||||
// Legacy first: its replays are mostly already on disk from ss-scraper.
|
||||
for (const id of [...PLAYER_IDS].reverse()) await exportSSReplays(id, since);
|
||||
return 0;
|
||||
case 'maps':
|
||||
await exportMaps(mapHashes(mapsSince));
|
||||
return 0;
|
||||
case 'all':
|
||||
await exportBL(since);
|
||||
await exportSS(since);
|
||||
await exportMaps(mapHashes(mapsSince));
|
||||
console.log('');
|
||||
report(since, mapsSince);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
if (cmd === 'report') {
|
||||
report(since, mapsSince);
|
||||
return 0;
|
||||
}
|
||||
|
||||
console.error('Usage: node scripts/archive/cli.ts <auth login|refresh|status | export bl|ss|ss-replays|maps|all | report> [--since YYYY-MM-DD|all] [--maps-since YYYY-MM-DD|all]');
|
||||
return 2;
|
||||
}
|
||||
|
||||
main().then(
|
||||
(code) => process.exit(code),
|
||||
(err) => {
|
||||
console.error(err instanceof Error ? err.message : err);
|
||||
process.exit(1);
|
||||
}
|
||||
);
|
||||
Reference in New Issue
Block a user