#!/usr/bin/env node // plebleader archive CLI. Run from the repo root: node scripts/archive/cli.ts [options] // // auth login | refresh | status the sync's own BeatLeader OAuth token chain // export bl [--since D] BL profile, scores, replays, score stats, leaderboards // export ss [--since D] ScoreSaber player snapshot and scores (fast) // export ss-replays [--since D] ScoreSaber replays, ~10.5 s apart (run in background) // export maps [--maps-since D] BeatSaver JSON + zip for played hashes (default: last 2 years, // plus all legacy ScoreSaber maps) // export all [--since D] bl, ss, maps (not ss-replays) // report [--since D] counts and gaps in the archive // // --since: YYYY-MM-DD, unix seconds, or "all" (default: all). --maps-since defaults to the two-year // window (DEFAULT_MAPS_SINCE). Every command skips files that already exist, so reruns only fill gaps. import * as fs from 'node:fs'; import * as path from 'node:path'; import { ARCHIVE_DIR, BL_API, DEFAULT_MAPS_SINCE, LEGACY_PLAYER_ID, MAIN_PLAYER_ID, PLAYER_IDS, TOKENS_FILE, USER_AGENT, loadDotEnv, parseSince } from './config.ts'; import { getAccessToken, login, readTokens, refresh } from './auth.ts'; import { archivedScores, blHashes, exportLeaderboards, exportProfile, exportReplays, exportScoreStats, exportScores } from './beatleader.ts'; import { archivedSSScores, exportSSPlayer, exportSSReplays, exportSSScores, ssHashes } from './scoresaber.ts'; import { exportMaps } from './beatsaver.ts'; import { archivePath, exists, listFiles, readState } from './store.ts'; function fmt(ms: number): string { const d = new Date(ms); const left = ms - Date.now(); const hours = Math.round(left / 36e5); return `${d.toISOString()} (${left < 0 ? 'expired' : hours >= 48 ? `${Math.round(hours / 24)} d left` : `${hours} h left`})`; } async function blGet(path: string, token: string): Promise { const res = await fetch(`${BL_API}${path}`, { headers: { Authorization: `Bearer ${token}`, 'User-Agent': USER_AGENT } }); if (!res.ok) throw new Error(`GET ${path}: HTTP ${res.status}`); return (await res.json()) as T; } /** * Phase 1 step 1. The bearer token must belong to pleb, and an unranked map's ratings must be * readable through /leaderboards/hash/{hash}. * * Verified 2026-11-08: BeatLeader only honors bearer tokens on /oauth2/identity, clan management * and ReeSabers presets. /user and the showAllRatings gate on score lists read the beatleader.com * cookie session instead, so score lists always come back with unranked ratings nulled. The * per-hash leaderboard lookup returns full ratings (incl. modifiersRating) even anonymously. */ async function verify(): Promise { const token = await getAccessToken(); const identity = await blGet<{ id?: string; name?: string }>('/oauth2/identity', token); const idOk = identity.id === MAIN_PLAYER_ID; console.log(`GET /oauth2/identity: ${identity.id} (${identity.name}) ${idOk ? 'OK' : `MISMATCH, expected ${MAIN_PLAYER_ID}`}`); type ScorePage = { data: { leaderboard: { song: { name: string; hash: string }; difficulty: { difficultyName: string; status: number } } }[] }; // status 3 = ranked; anything else counts as unranked for this check. const page = await blGet(`/player/${MAIN_PLAYER_ID}/scores?count=50&page=1&sortBy=date&order=desc&type=unranked`, token); const unranked = page.data.find((s) => s.leaderboard.difficulty.status !== 3); if (!unranked) { console.log('No unranked score found in the latest 50 to check ratings against.'); return idOk; } type HashLookup = { leaderboards: { difficulty: { difficultyName: string; stars: number | null; techRating: number | null } }[] }; const { song, difficulty } = unranked.leaderboard; const lookup = await blGet(`/leaderboards/hash/${song.hash}`, token); const d = lookup.leaderboards.find((l) => l.difficulty.difficultyName === difficulty.difficultyName)?.difficulty; const ratingsOk = d?.stars != null; console.log(`Unranked "${song.name}" ${difficulty.difficultyName} via /leaderboards/hash: stars=${d?.stars} tech=${d?.techRating} ${ratingsOk ? 'OK' : 'MISSING'}`); return idOk && ratingsOk; } function playedHashes(since: number): string[] { return [...new Set([...blHashes(since), ...PLAYER_IDS.flatMap((id) => ssHashes(id, since))])].sort(); } /** * Hashes to fetch BeatSaver map files for: played within the maps window, plus — regardless of * age — the maps of legacy ScoreSaber scores that have an archived replay, so those replays render. */ function mapHashes(mapsSince: number): string[] { const legacyWithReplay = archivedSSScores(LEGACY_PLAYER_ID, 0) .filter((it) => exists(archivePath('scoresaber', LEGACY_PLAYER_ID, 'replays', `${it.score.id}.dat`))) .map((it) => it.leaderboard.songHash.toLowerCase()); return [...new Set([...playedHashes(mapsSince), ...legacyWithReplay])].sort(); } async function exportBL(since: number): Promise { for (const id of PLAYER_IDS) { await exportProfile(id); await exportScores(id, since); } // The scores dir holds both accounts' scores (keyed by score id). const scores = archivedScores(since); await exportReplays(scores); await exportScoreStats(scores); // BL leaderboard JSON (ratings) for every played hash, not just the maps window: it's small // and the importer needs it for star ratings on old scores. await exportLeaderboards(blHashes(since)); } async function exportSS(since: number): Promise { for (const id of PLAYER_IDS) { await exportSSPlayer(id); await exportSSScores(id, since); } // Hashes only ScoreSaber knows about still need BL leaderboard data (phase 5 merge). await exportLeaderboards([...new Set(PLAYER_IDS.flatMap((id) => ssHashes(id, since)))].sort()); } function report(since: number, mapsSince: number): void { const count = (...p: string[]) => listFiles(archivePath(...p)).length; const day = (t: number) => (t ? new Date(t * 1000).toISOString().slice(0, 10) : 'all time'); const bl = archivedScores(since); const blWithReplay = bl.filter((s) => s.replay); const blReplayMissing = blWithReplay.filter((s) => !exists(archivePath('beatleader', 'replays', decodeURIComponent(new URL(s.replay!).pathname.split('/').pop()!)))); const statsMissing = bl.filter((s) => !exists(archivePath('beatleader', 'scorestats', `${s.id}.json`))); const ss = PLAYER_IDS.map((id) => { const items = archivedSSScores(id, since); const withReplay = items.filter((it) => it.score.hasReplay); const missing = withReplay.filter((it) => !exists(archivePath('scoresaber', id, 'replays', `${it.score.id}.dat`))); return { id, items, withReplay, missing }; }); const allHashes = playedHashes(since); const noLeaderboard = allHashes.filter((h) => listFiles(archivePath('beatleader', 'leaderboards', h)).length === 0); const hashes = mapHashes(Math.max(since, mapsSince)); const noZip = hashes.filter((h) => !exists(archivePath('beatsaver', 'zips', `${h}.zip`))); let missingJson: Record = {}; try { missingJson = JSON.parse(fs.readFileSync(archivePath('beatsaver', 'missing.json'), 'utf-8')); } catch {} const du = (dir: string): number => { let total = 0; const walk = (d: string) => { for (const e of fs.readdirSync(d, { withFileTypes: true })) { const p = path.join(d, e.name); if (e.isDirectory()) walk(p); else total += fs.statSync(p).size; } }; if (fs.existsSync(dir)) walk(dir); return total; }; console.log(`Archive: ${ARCHIVE_DIR}`); console.log(`Window: scores since ${day(since)}, BeatSaver maps since ${day(mapsSince)}`); console.log(''); console.log(`BeatLeader scores ${bl.length} in window (${count('beatleader', 'scores')} archived total, both accounts)`); console.log(`BeatLeader replays ${blWithReplay.length - blReplayMissing.length}/${blWithReplay.length}`); console.log(`BeatLeader score stats ${bl.length - statsMissing.length}/${bl.length}`); for (const s of ss) { console.log(`ScoreSaber ${s.id}: ${s.items.length} scores, replays ${s.withReplay.length - s.missing.length}/${s.withReplay.length}`); } console.log(`Played hashes ${allHashes.length}`); console.log(` BL leaderboard snapshot ${allHashes.length - noLeaderboard.length}/${allHashes.length}`); console.log(` BeatSaver zip ${hashes.length - noZip.length}/${hashes.length} in maps window (missing.json lists ${Object.keys(missingJson).length})`); console.log(`Archive size ${(du(ARCHIVE_DIR) / 1e9).toFixed(2)} GB`); const failures = readState().failures; if (failures && Object.keys(failures).length) console.log(`Failures from last runs: ${JSON.stringify(failures)}`); } function option(args: string[], name: string): string | undefined { const i = args.indexOf(name); return i >= 0 ? args[i + 1] : args.find((a) => a.startsWith(`${name}=`))?.slice(name.length + 1); } async function main(): Promise { loadDotEnv(); const args = process.argv.slice(2); const [cmd, sub] = args; if (cmd === 'auth' && sub === 'login') { const store = await login(); console.log(`\nLogged in as ${store.player_name} (${store.player_id}). Tokens saved to ${TOKENS_FILE}`); console.log(`Refresh token valid until about ${fmt(store.refresh_expires_at)}`); return (await verify()) ? 0 : 1; } if (cmd === 'auth' && sub === 'refresh') { const store = await refresh(); console.log(`Refreshed. Access token until ${fmt(store.access_expires_at)}; refresh until about ${fmt(store.refresh_expires_at)}`); return 0; } if (cmd === 'auth' && sub === 'status') { const store = readTokens(); if (!store) { console.log(`No tokens at ${TOKENS_FILE}. Run \`auth login\`.`); return 1; } console.log(`Tokens: ${TOKENS_FILE}`); console.log(`Player: ${store.player_name} (${store.player_id})`); console.log(`Access: ${fmt(store.access_expires_at)}`); console.log(`Refresh: ${fmt(store.refresh_expires_at)} (estimated)`); return (await verify()) ? 0 : 1; } const since = parseSince(option(args, '--since')); const mapsSince = Math.max(since, parseSince(option(args, '--maps-since'), DEFAULT_MAPS_SINCE)); if (cmd === 'export') { console.log(`Archive ${ARCHIVE_DIR}, since ${since ? new Date(since * 1000).toISOString() : 'all time'}\n`); switch (sub) { case 'bl': await exportBL(since); return 0; case 'ss': await exportSS(since); return 0; case 'ss-replays': // Legacy first: its replays are mostly already on disk from ss-scraper. for (const id of [...PLAYER_IDS].reverse()) await exportSSReplays(id, since); return 0; case 'maps': await exportMaps(mapHashes(mapsSince)); return 0; case 'all': await exportBL(since); await exportSS(since); await exportMaps(mapHashes(mapsSince)); console.log(''); report(since, mapsSince); return 0; } } if (cmd === 'report') { report(since, mapsSince); return 0; } console.error('Usage: node scripts/archive/cli.ts [--since YYYY-MM-DD|all] [--maps-since YYYY-MM-DD|all]'); return 2; } main().then( (code) => process.exit(code), (err) => { console.error(err instanceof Error ? err.message : err); process.exit(1); } );