Record the search index rebuild schedule in the plan

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.gitignore

@@ -24,4 +24,4 @@ vite.config.ts.timestamp-*
 .data
 
 .env
-archive
+/archive

AGENTS.md

@@ -17,3 +17,7 @@
 
 - Dont' use `cd`, prefer `pwd`
 - Dont' use `rg`, prefer `grep -r`
+
+## Personal BeatLeader mirror
+
+- Active project: a self-hosted BeatLeader fork on incineroar holding only pleb's data, plus ScoreSaber history and ArcViewer. Read [docs/plan.md](docs/plan.md) first; it records the decisions, the verified API facts, and the phased steps.

flake.nix

@@ -44,12 +44,35 @@
           mainProgram = "plebsaber-stream";
         };
       };
+    # The plebleader archive CLI (scripts/archive, docs/plan.md): plain
+    # TypeScript with only Node built-ins, run through Node's type stripping.
+    mkArchiveCli = pkgs:
+      pkgs.stdenvNoCC.mkDerivation {
+        pname = "plebleader-archive";
+        version = "0.1.0";
+        src = ./scripts/archive;
+        nativeBuildInputs = [ pkgs.makeWrapper ];
+        installPhase = ''
+          runHook preInstall
+          mkdir -p $out/lib/plebleader-archive
+          cp *.ts $out/lib/plebleader-archive/
+          makeWrapper ${pkgs.nodejs_22}/bin/node $out/bin/plebleader-archive \
+            --add-flags "--experimental-strip-types --disable-warning=ExperimentalWarning" \
+            --add-flags "$out/lib/plebleader-archive/cli.ts"
+          runHook postInstall
+        '';
+        meta = {
+          description = "Archive exporter and daily sync for the personal BeatLeader mirror";
+          mainProgram = "plebleader-archive";
+        };
+      };
   in rec {
     packages = forAllSystems (system: let
       pkgs = import nixpkgs { inherit system; };
       pkg = mkPackage pkgs;
     in {
       plebsaber-stream = pkg;
+      plebleader-archive = mkArchiveCli pkgs;
       default = pkg;
     });
 

scripts/archive/auth.ts

@@ -0,0 +1,192 @@
+// BeatLeader OAuth 2.0 token chain owned by the archive sync.
+//
+// This chain is separate from plebsaber.stream's own login: BeatLeader rotates
+// the refresh token on every refresh and invalidates the old one, so two
+// consumers sharing one chain would break each other.
+import * as fs from 'node:fs';
+import * as http from 'node:http';
+import { randomUUID } from 'node:crypto';
+import { BL_API, TOKENS_FILE, USER_AGENT, clientCredentials, writeFileAtomic } from './config.ts';
+
+const AUTHORIZE_URL = `${BL_API}/oauth2/authorize`;
+const TOKEN_URL = `${BL_API}/oauth2/token`;
+const SCOPES = 'profile offline_access';
+// Registered redirect URIs on the OAuth app (verified 2026-11-08):
+//   https://plebsaber.stream/auth/beatleader/callback
+//   http://localhost:5173/auth/beatleader/callback
+const LOOPBACK_PORT = 5173;
+const LOOPBACK_REDIRECT = `http://localhost:${LOOPBACK_PORT}/auth/beatleader/callback`;
+const REFRESH_TOKEN_LIFETIME_MS = 14 * 24 * 3600 * 1000;
+
+export type TokenStore = {
+	access_token: string;
+	refresh_token: string;
+	/** epoch ms */
+	access_expires_at: number;
+	/** epoch ms, estimated from BeatLeader's 14-day refresh token lifetime */
+	refresh_expires_at: number;
+	/** epoch ms of the last token grant/refresh */
+	updated_at: number;
+	player_id?: string;
+	player_name?: string;
+};
+
+type TokenResponse = { access_token: string; refresh_token?: string; expires_in?: number; token_type?: string };
+
+export function readTokens(): TokenStore | null {
+	try {
+		return JSON.parse(fs.readFileSync(TOKENS_FILE, 'utf-8')) as TokenStore;
+	} catch {
+		return null;
+	}
+}
+
+function saveTokens(res: TokenResponse, previous: TokenStore | null, identity?: { id: string; name?: string }): TokenStore {
+	if (!res.refresh_token) {
+		throw new Error('Token response has no refresh_token; is offline_access granted?');
+	}
+	const now = Date.now();
+	const store: TokenStore = {
+		access_token: res.access_token,
+		refresh_token: res.refresh_token,
+		access_expires_at: now + (res.expires_in ?? 3600) * 1000,
+		refresh_expires_at: now + REFRESH_TOKEN_LIFETIME_MS,
+		updated_at: now,
+		player_id: identity?.id ?? previous?.player_id,
+		player_name: identity?.name ?? previous?.player_name
+	};
+	// Persist the rotated refresh token before anything uses the new access token.
+	writeFileAtomic(TOKENS_FILE, JSON.stringify(store, null, 2) + '\n', 0o600);
+	return store;
+}
+
+async function tokenRequest(params: Record<string, string>): Promise<TokenResponse> {
+	const { clientId, clientSecret } = clientCredentials();
+	const res = await fetch(TOKEN_URL, {
+		method: 'POST',
+		headers: { 'Content-Type': 'application/x-www-form-urlencoded', 'User-Agent': USER_AGENT },
+		body: new URLSearchParams({ client_id: clientId, client_secret: clientSecret, ...params })
+	});
+	const text = await res.text();
+	if (!res.ok) throw new Error(`Token request failed: HTTP ${res.status} ${text.slice(0, 300)}`);
+	return JSON.parse(text) as TokenResponse;
+}
+
+export async function fetchIdentity(accessToken: string): Promise<{ id: string; name?: string }> {
+	const res = await fetch(`${BL_API}/oauth2/identity`, {
+		headers: { Authorization: `Bearer ${accessToken}`, 'User-Agent': USER_AGENT }
+	});
+	if (!res.ok) throw new Error(`Identity request failed: HTTP ${res.status}`);
+	const json = (await res.json()) as { id?: string; name?: string };
+	if (!json.id) throw new Error('Identity response has no id');
+	return { id: json.id, name: json.name };
+}
+
+/** Wait for the OAuth redirect on the loopback port, or for a pasted redirect URL on stdin. */
+function waitForCode(state: string): Promise<string> {
+	return new Promise((resolve, reject) => {
+		let done = false;
+		const finish = (err: Error | null, code?: string) => {
+			if (done) return;
+			done = true;
+			server.close();
+			process.stdin.pause();
+			if (err) reject(err);
+			else resolve(code!);
+		};
+
+		const fromUrl = (raw: string): string => {
+			const url = new URL(raw, LOOPBACK_REDIRECT);
+			const error = url.searchParams.get('error');
+			if (error) throw new Error(`Authorization denied: ${error} ${url.searchParams.get('error_description') ?? ''}`);
+			const code = url.searchParams.get('code');
+			if (!code) throw new Error('Redirect has no code parameter');
+			if (url.searchParams.get('state') !== state) throw new Error('State mismatch; start the login again');
+			return code;
+		};
+
+		const server = http.createServer((req, res) => {
+			if (!req.url?.startsWith('/auth/beatleader/callback')) {
+				res.writeHead(404).end();
+				return;
+			}
+			try {
+				const code = fromUrl(req.url);
+				res.writeHead(200, { 'Content-Type': 'text/plain; charset=utf-8' });
+				res.end('plebleader archive: BeatLeader authorization received. You can close this tab.\n');
+				finish(null, code);
+			} catch (err) {
+				res.writeHead(400, { 'Content-Type': 'text/plain; charset=utf-8' });
+				res.end(`plebleader archive: ${(err as Error).message}\n`);
+				finish(err as Error);
+			}
+		});
+		server.on('error', (err: NodeJS.ErrnoException) => {
+			if (err.code === 'EADDRINUSE') {
+				console.error(`Port ${LOOPBACK_PORT} is busy (is \`vite dev\` running?). Paste the redirected URL below instead.`);
+			} else {
+				finish(err);
+			}
+		});
+		server.listen(LOOPBACK_PORT, 'localhost');
+
+		// Fallback: paste the full redirected URL (e.g. when the browser is on another machine).
+		process.stdin.setEncoding('utf-8');
+		process.stdin.on('data', (chunk: string) => {
+			const line = chunk.trim();
+			if (!line) return;
+			try {
+				finish(null, fromUrl(line));
+			} catch (err) {
+				console.error((err as Error).message);
+			}
+		});
+		process.stdin.resume();
+	});
+}
+
+export async function login(): Promise<TokenStore> {
+	const { clientId } = clientCredentials();
+	const state = randomUUID();
+	const url = new URL(AUTHORIZE_URL);
+	url.searchParams.set('client_id', clientId);
+	url.searchParams.set('scope', SCOPES);
+	url.searchParams.set('response_type', 'code');
+	url.searchParams.set('redirect_uri', LOOPBACK_REDIRECT);
+	url.searchParams.set('prompt', 'consent');
+	url.searchParams.set('state', state);
+
+	console.log('Open this URL in a browser on this machine and approve access:\n');
+	console.log(`  ${url.toString()}\n`);
+	console.log(`Listening on ${LOOPBACK_REDIRECT}`);
+	console.log('If the browser is on another machine, paste the URL it was redirected to (the page will fail to load) and press Enter.\n');
+
+	const code = await waitForCode(state);
+	const res = await tokenRequest({ grant_type: 'authorization_code', code, redirect_uri: LOOPBACK_REDIRECT });
+	// Save immediately so the refresh token survives even if the identity lookup fails.
+	let store = saveTokens(res, readTokens());
+	const identity = await fetchIdentity(store.access_token);
+	store = { ...store, player_id: identity.id, player_name: identity.name };
+	writeFileAtomic(TOKENS_FILE, JSON.stringify(store, null, 2) + '\n', 0o600);
+	return store;
+}
+
+export async function refresh(): Promise<TokenStore> {
+	const current = readTokens();
+	if (!current?.refresh_token) {
+		throw new Error(`No refresh token in ${TOKENS_FILE}. Run \`auth login\` first.`);
+	}
+	try {
+		const res = await tokenRequest({ grant_type: 'refresh_token', refresh_token: current.refresh_token });
+		return saveTokens(res, current);
+	} catch (err) {
+		throw new Error(`${(err as Error).message}\nThe refresh chain is broken. Run \`node scripts/archive/cli.ts auth login\` to seed it again.`);
+	}
+}
+
+/** A valid access token, refreshing (and rotating the stored refresh token) when needed. */
+export async function getAccessToken(minValidityMs = 5 * 60 * 1000): Promise<string> {
+	const current = readTokens();
+	if (current && current.access_expires_at - Date.now() > minValidityMs) return current.access_token;
+	return (await refresh()).access_token;
+}

scripts/archive/beatleader.ts

@@ -0,0 +1,124 @@
+// BeatLeader exporter: scores, replays, score stats, leaderboards per played hash, profile.
+// Writes only to archive/beatleader/. Everything is resumable: existing files are skipped.
+import * as path from 'node:path';
+import { BL_API } from './config.ts';
+import { get, limiters } from './http.ts';
+import { archivePath, eachWithProgress, exists, listFiles, readJson, readState, save, saveSnapshotIfChanged, today, updateState } from './store.ts';
+
+const bl = (...p: string[]) => archivePath('beatleader', ...p);
+
+export type BLScore = { id: number; playerId: string; timepost: number; replay?: string | null; leaderboard: { song: { hash: string } } };
+
+async function blGet(pathAndQuery: string, accept?: number[]) {
+	return get(`${BL_API}${pathAndQuery}`, { limiter: limiters.beatleader, accept });
+}
+
+/** Archived BL scores (parsed), optionally only those at or after `sinceUnix`. */
+export function archivedScores(sinceUnix = 0): BLScore[] {
+	const dir = bl('scores');
+	return listFiles(dir, '.json')
+		.map((f) => readJson<BLScore>(path.join(dir, f)))
+		.filter((s) => s.timepost >= sinceUnix);
+}
+
+export async function exportProfile(playerId: string): Promise<void> {
+	console.log(`BeatLeader profile ${playerId}`);
+	const dir = (kind: string) => bl('players', playerId, kind);
+	const endpoints: [string, string][] = [
+		['profile', `/player/${playerId}?stats=true`],
+		// count: days of history; the default is 50, which loses the triangle's
+		// monthly points. Upstream keeps one row per day since the first score.
+		['history', `/player/${playerId}/history?count=100000`],
+		['triangle', `/player/${playerId}/history/triangle`],
+		['accgraph', `/player/${playerId}/accgraph`]
+	];
+	for (const [kind, url] of endpoints) {
+		const res = await blGet(url);
+		const wrote = saveSnapshotIfChanged(dir(kind), res.body);
+		console.log(`  ${kind}: ${wrote ? `saved ${today()}` : 'unchanged'}`);
+	}
+}
+
+/** Page through best scores newest-first from `sinceUnix`, saving each score object. Returns the new ones. */
+export async function exportScores(playerId: string, sinceUnix: number): Promise<BLScore[]> {
+	console.log(`BeatLeader scores for ${playerId} since ${new Date(sinceUnix * 1000).toISOString()}`);
+	const added: BLScore[] = [];
+	let seen = 0;
+	let newest = Number(readState()[`beatleader:${playerId}`]?.newestTimepost ?? 0);
+	for (let page = 1; ; page++) {
+		const qs = `count=100&page=${page}&sortBy=date&order=desc&includeIO=true&time_from=${sinceUnix}`;
+		const res = await blGet(`/player/${playerId}/scores?${qs}`);
+		const json = JSON.parse(res.body.toString('utf-8')) as { metadata: { total: number }; data: BLScore[] };
+		for (const score of json.data) {
+			seen++;
+			newest = Math.max(newest, score.timepost);
+			const file = bl('scores', `${score.id}.json`);
+			if (exists(file)) continue;
+			// The page is parsed and each element re-serialized: same content, no reshaping.
+			save(file, JSON.stringify(score));
+			added.push(score);
+		}
+		console.log(`  page ${page}: ${seen}/${json.metadata.total} seen, ${added.length} new`);
+		if (json.data.length < 100 || seen >= json.metadata.total) break;
+	}
+	updateState(`beatleader:${playerId}`, { scoresSince: sinceUnix, newestTimepost: newest, lastScoresRun: new Date().toISOString() });
+	return added;
+}
+
+/** Newest archived score time for a player (unix seconds), 0 when there is none. */
+export function newestScoreTime(playerId: string): number {
+	return archivedScores().reduce((max, s) => (s.playerId === playerId ? Math.max(max, s.timepost) : max), 0);
+}
+
+export async function exportReplays(scores: BLScore[]): Promise<void> {
+	const withReplay = scores.filter((s) => s.replay);
+	console.log(`BeatLeader replays: ${withReplay.length} scores with a replay URL`);
+	const res = await eachWithProgress('replays', withReplay, async (s) => {
+		const name = decodeURIComponent(new URL(s.replay!).pathname.split('/').pop()!);
+		const file = bl('replays', name);
+		if (exists(file)) return 'skipped';
+		const r = await get(s.replay!, { limiter: limiters.blCdn, accept: [404] });
+		if (r.status === 404) return 'missing';
+		save(file, r.body);
+		return 'done';
+	});
+	recordFailures('replays', res.failed.map((f) => f.item.id));
+}
+
+export async function exportScoreStats(scores: BLScore[]): Promise<void> {
+	console.log(`BeatLeader score stats: ${scores.length} scores`);
+	const res = await eachWithProgress('scorestats', scores, async (s) => {
+		const file = bl('scorestats', `${s.id}.json`);
+		if (exists(file)) return 'skipped';
+		// 302 to a presigned R2 URL; fetch follows it.
+		const r = await blGet(`/score/statistic/${s.id}`, [404]);
+		if (r.status === 404) return 'missing';
+		save(file, r.body);
+		return 'done';
+	});
+	recordFailures('scorestats', res.failed.map((f) => f.item.id));
+}
+
+/** /leaderboards/hash/{hash}: all difficulties with full ratings (also unranked, no auth needed). */
+export async function exportLeaderboards(hashes: string[], refresh = false): Promise<void> {
+	console.log(`BeatLeader leaderboards: ${hashes.length} hashes${refresh ? ' (refresh)' : ''}`);
+	const res = await eachWithProgress('leaderboards', hashes, async (hash) => {
+		const dir = bl('leaderboards', hash);
+		if (!refresh && listFiles(dir, '.json').length > 0) return 'skipped';
+		const r = await blGet(`/leaderboards/hash/${hash}`, [404]);
+		if (r.status === 404) return 'missing';
+		return saveSnapshotIfChanged(dir, r.body) ? 'done' : 'skipped';
+	});
+	recordFailures('leaderboards', res.failed.map((f) => f.item));
+}
+
+function recordFailures(kind: string, ids: (string | number)[]): void {
+	if (ids.length) {
+		updateState('failures', { [`beatleader:${kind}`]: ids });
+		console.warn(`  ${ids.length} ${kind} failed; rerun to retry`);
+	}
+}
+
+export function blHashes(sinceUnix = 0): string[] {
+	return [...new Set(archivedScores(sinceUnix).map((s) => s.leaderboard.song.hash.toLowerCase()))].sort();
+}

scripts/archive/beatsaver.ts

@@ -0,0 +1,89 @@
+// BeatSaver exporter: map JSON and the zip of the exact version hash that was played.
+import * as fs from 'node:fs';
+import { get, limiters } from './http.ts';
+import { archivePath, eachWithProgress, exists, readJson, save } from './store.ts';
+
+const bs = (...p: string[]) => archivePath('beatsaver', ...p);
+const MISSING_FILE = () => bs('missing.json');
+
+type BSMap = { id: string; versions: { hash: string; downloadURL: string }[] };
+type Missing = Record<string, { reason: string; checkedAt: string; recoveredFrom?: string }>;
+
+function readMissing(): Missing {
+	try {
+		return readJson<Missing>(MISSING_FILE());
+	} catch {
+		return {};
+	}
+}
+
+/** Entry names from a zip's central directory, or null if the zip is truncated or malformed. */
+export function zipEntryNames(buf: Buffer): string[] | null {
+	if (buf.length < 22 || buf.readUInt32LE(0) !== 0x04034b50) return null;
+	const tailStart = Math.max(0, buf.length - 65_557);
+	const eocd = buf.subarray(tailStart).lastIndexOf(Buffer.from([0x50, 0x4b, 0x05, 0x06]));
+	if (eocd < 0) return null;
+	const e = tailStart + eocd;
+	const count = buf.readUInt16LE(e + 10);
+	let p = buf.readUInt32LE(e + 16);
+	const names: string[] = [];
+	for (let i = 0; i < count; i++) {
+		if (p + 46 > buf.length || buf.readUInt32LE(p) !== 0x02014b50) return null;
+		const nameLen = buf.readUInt16LE(p + 28);
+		const extraLen = buf.readUInt16LE(p + 30);
+		const commentLen = buf.readUInt16LE(p + 32);
+		names.push(buf.toString('utf-8', p + 46, p + 46 + nameLen));
+		p += 46 + nameLen + extraLen + commentLen;
+	}
+	return names;
+}
+
+/** A zip is usable if its central directory is intact and lists an Info.dat (any case, any folder). */
+export function looksLikeMapZip(buf: Buffer): boolean {
+	return zipEntryNames(buf)?.some((n) => /(^|\/)info\.dat$/i.test(n)) ?? false;
+}
+
+export async function exportMaps(hashes: string[]): Promise<void> {
+	const missing = readMissing();
+	console.log(`BeatSaver maps: ${hashes.length} hashes`);
+	const res = await eachWithProgress('maps', hashes, async (hash) => {
+		const jsonFile = bs('maps', `${hash}.json`);
+		const zipFile = bs('zips', `${hash}.zip`);
+		if (exists(zipFile) && exists(jsonFile)) return 'skipped';
+
+		let map: BSMap | null = null;
+		if (exists(jsonFile)) {
+			map = readJson<BSMap>(jsonFile);
+		} else {
+			const r = await get(`https://api.beatsaver.com/maps/hash/${hash}`, { limiter: limiters.beatsaver, accept: [404] });
+			if (r.status === 200) {
+				save(jsonFile, r.body);
+				map = JSON.parse(r.body.toString('utf-8')) as BSMap;
+			}
+		}
+		if (exists(zipFile)) return 'done';
+
+		// Prefer the version entry for this exact hash; old versions often remain on the CDN.
+		const version = map?.versions.find((v) => v.hash.toLowerCase() === hash);
+		const candidates = [version?.downloadURL, `https://r2cdn.beatsaver.com/${hash}.zip`, `https://cdn.beatsaver.com/${hash}.zip`].filter(
+			(u): u is string => Boolean(u)
+		);
+		for (const url of [...new Set(candidates)]) {
+			const r = await get(url, { limiter: limiters.beatsaverCdn, accept: [403, 404] });
+			if (r.status === 200 && looksLikeMapZip(r.body)) {
+				save(zipFile, r.body);
+				delete missing[hash];
+				return 'done';
+			}
+		}
+		missing[hash] = {
+			reason: map ? `map ${map.id} found but version zip unavailable` : 'hash unknown to BeatSaver',
+			checkedAt: new Date().toISOString()
+		};
+		return 'missing';
+	});
+	fs.mkdirSync(bs(), { recursive: true });
+	save(MISSING_FILE(), JSON.stringify(missing, null, 2) + '\n');
+	console.log(`  missing.json: ${Object.keys(missing).length} hashes`);
+	if (res.failed.length) console.warn(`  ${res.failed.length} failed; rerun to retry`);
+}

scripts/archive/cli.ts

@@ -0,0 +1,303 @@
+#!/usr/bin/env node
+// plebleader archive CLI. Run from the repo root: node scripts/archive/cli.ts <command> [options]
+//
+//   auth login | refresh | status   the sync's own BeatLeader OAuth token chain
+//   export bl          [--since D]  BL profile, scores, replays, score stats, leaderboards
+//   export ss          [--since D]  ScoreSaber player snapshot and scores (fast)
+//   export ss-replays  [--since D]  ScoreSaber replays, ~10.5 s apart (run in background)
+//   export maps   [--maps-since D]  BeatSaver JSON + zip for played hashes (default: last 2 years,
+//                                   plus all legacy ScoreSaber maps)
+//   export all         [--since D]  bl, ss, maps (not ss-replays)
+//   report             [--since D]  counts and gaps in the archive
+//   sync    [--ratings] [--no-ss-replays]
+//                                   daily incremental run (phase 6): token refresh, new BL and SS
+//                                   scores with their replays, stats, leaderboards and maps.
+//                                   --ratings re-snapshots every played hash (default on Sundays)
+//
+// --since: YYYY-MM-DD, unix seconds, or "all" (default: all). --maps-since defaults to the two-year
+// window (DEFAULT_MAPS_SINCE). Every command skips files that already exist, so reruns only fill gaps.
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { ARCHIVE_DIR, BL_API, DEFAULT_MAPS_SINCE, LEGACY_PLAYER_ID, MAIN_PLAYER_ID, PLAYER_IDS, TOKENS_FILE, USER_AGENT, loadDotEnv, parseSince } from './config.ts';
+import { getAccessToken, login, readTokens, refresh } from './auth.ts';
+import { archivedScores, blHashes, exportLeaderboards, exportProfile, exportReplays, exportScoreStats, exportScores, newestScoreTime } from './beatleader.ts';
+import { archivedSSScores, exportSSPlayer, exportSSReplays, exportSSScores, newestSSScoreTime, ssHashes, ssMapHash } from './scoresaber.ts';
+import { exportMaps } from './beatsaver.ts';
+import { archivePath, exists, listFiles, readState, updateState } from './store.ts';
+
+function fmt(ms: number): string {
+	const d = new Date(ms);
+	const left = ms - Date.now();
+	const hours = Math.round(left / 36e5);
+	return `${d.toISOString()} (${left < 0 ? 'expired' : hours >= 48 ? `${Math.round(hours / 24)} d left` : `${hours} h left`})`;
+}
+
+async function blGet<T>(path: string, token: string): Promise<T> {
+	const res = await fetch(`${BL_API}${path}`, {
+		headers: { Authorization: `Bearer ${token}`, 'User-Agent': USER_AGENT }
+	});
+	if (!res.ok) throw new Error(`GET ${path}: HTTP ${res.status}`);
+	return (await res.json()) as T;
+}
+
+/**
+ * Phase 1 step 1. The bearer token must belong to pleb, and an unranked map's ratings must be
+ * readable through /leaderboards/hash/{hash}.
+ *
+ * Verified 2026-11-08: BeatLeader only honors bearer tokens on /oauth2/identity, clan management
+ * and ReeSabers presets. /user and the showAllRatings gate on score lists read the beatleader.com
+ * cookie session instead, so score lists always come back with unranked ratings nulled. The
+ * per-hash leaderboard lookup returns full ratings (incl. modifiersRating) even anonymously.
+ */
+async function verify(): Promise<boolean> {
+	const token = await getAccessToken();
+	const identity = await blGet<{ id?: string; name?: string }>('/oauth2/identity', token);
+	const idOk = identity.id === MAIN_PLAYER_ID;
+	console.log(`GET /oauth2/identity: ${identity.id} (${identity.name}) ${idOk ? 'OK' : `MISMATCH, expected ${MAIN_PLAYER_ID}`}`);
+
+	type ScorePage = { data: { leaderboard: { song: { name: string; hash: string }; difficulty: { difficultyName: string; status: number } } }[] };
+	// status 3 = ranked; anything else counts as unranked for this check.
+	const page = await blGet<ScorePage>(`/player/${MAIN_PLAYER_ID}/scores?count=50&page=1&sortBy=date&order=desc&type=unranked`, token);
+	const unranked = page.data.find((s) => s.leaderboard.difficulty.status !== 3);
+	if (!unranked) {
+		console.log('No unranked score found in the latest 50 to check ratings against.');
+		return idOk;
+	}
+	type HashLookup = { leaderboards: { difficulty: { difficultyName: string; stars: number | null; techRating: number | null } }[] };
+	const { song, difficulty } = unranked.leaderboard;
+	const lookup = await blGet<HashLookup>(`/leaderboards/hash/${song.hash}`, token);
+	const d = lookup.leaderboards.find((l) => l.difficulty.difficultyName === difficulty.difficultyName)?.difficulty;
+	const ratingsOk = d?.stars != null;
+	console.log(`Unranked "${song.name}" ${difficulty.difficultyName} via /leaderboards/hash: stars=${d?.stars} tech=${d?.techRating} ${ratingsOk ? 'OK' : 'MISSING'}`);
+	return idOk && ratingsOk;
+}
+
+function playedHashes(since: number): string[] {
+	return [...new Set([...blHashes(since), ...PLAYER_IDS.flatMap((id) => ssHashes(id, since))])].sort();
+}
+
+/**
+ * Hashes to fetch BeatSaver map files for: played within the maps window, plus — regardless of
+ * age — the maps of legacy ScoreSaber scores that have an archived replay, so those replays render.
+ */
+function mapHashes(mapsSince: number): string[] {
+	const legacyWithReplay = archivedSSScores(LEGACY_PLAYER_ID, 0)
+		.filter((it) => exists(archivePath('scoresaber', LEGACY_PLAYER_ID, 'replays', `${it.score.id}.dat`)))
+		.map(ssMapHash);
+	return [...new Set([...playedHashes(mapsSince), ...legacyWithReplay])].sort();
+}
+
+async function exportBL(since: number): Promise<void> {
+	for (const id of PLAYER_IDS) {
+		await exportProfile(id);
+		await exportScores(id, since);
+	}
+	// The scores dir holds both accounts' scores (keyed by score id).
+	const scores = archivedScores(since);
+	await exportReplays(scores);
+	await exportScoreStats(scores);
+	// BL leaderboard JSON (ratings) for every played hash, not just the maps window: it's small
+	// and the importer needs it for star ratings on old scores.
+	await exportLeaderboards(blHashes(since));
+}
+
+async function exportSS(since: number): Promise<void> {
+	for (const id of PLAYER_IDS) {
+		await exportSSPlayer(id);
+		await exportSSScores(id, since);
+	}
+	// Hashes only ScoreSaber knows about still need BL leaderboard data (phase 5 merge).
+	await exportLeaderboards([...new Set(PLAYER_IDS.flatMap((id) => ssHashes(id, since)))].sort());
+}
+
+/** Overlap of each incremental window, so scores posted late or with a skewed clock aren't missed. */
+const SYNC_OVERLAP_S = 24 * 3600;
+
+/**
+ * Phase 6: the daily run. Main account only; the legacy accounts are frozen. Every step skips
+ * what's already archived, so a rerun (or a run after a missed day) just fills the gap.
+ */
+async function sync(opts: { ratings: boolean; ssReplays: boolean }): Promise<number> {
+	const started = new Date();
+	// Token upkeep first: persist the rotated refresh token before anything else, and fail loudly.
+	const tokens = await refresh();
+	console.log(`Token refreshed; refresh chain valid until about ${fmt(tokens.refresh_expires_at)}\n`);
+
+	await exportProfile(MAIN_PLAYER_ID);
+	const blSince = Math.max(0, newestScoreTime(MAIN_PLAYER_ID) - SYNC_OVERLAP_S);
+	const newBL = await exportScores(MAIN_PLAYER_ID, blSince);
+	// The whole window, not just the new scores, so a replay or stats file that failed yesterday is retried.
+	const blWindow = archivedScores(blSince).filter((s) => s.playerId === MAIN_PLAYER_ID);
+	await exportReplays(blWindow);
+	await exportScoreStats(blWindow);
+	// A new score can change a leaderboard (status, ratings), so re-snapshot its hash.
+	const newBLHashes = [...new Set(newBL.map((s) => s.leaderboard.song.hash.toLowerCase()))].sort();
+	await exportLeaderboards(newBLHashes, true);
+
+	await exportSSPlayer(MAIN_PLAYER_ID);
+	const ssSince = Math.max(0, newestSSScoreTime(MAIN_PLAYER_ID) - SYNC_OVERLAP_S);
+	const newSS = await exportSSScores(MAIN_PLAYER_ID, ssSince);
+	const newSSHashes = [...new Set(newSS.map(ssMapHash))].sort();
+	await exportLeaderboards(newSSHashes.filter((h) => !newBLHashes.includes(h)));
+	if (opts.ssReplays) await exportSSReplays(MAIN_PLAYER_ID, ssSince);
+
+	// Maps for everything played in either window; existing zips are skipped.
+	const windowHashes = [...new Set([...blWindow.map((s) => s.leaderboard.song.hash.toLowerCase()), ...ssHashes(MAIN_PLAYER_ID, ssSince)])].sort();
+	await exportMaps(windowHashes);
+	const noZip = windowHashes.filter((h) => !exists(archivePath('beatsaver', 'zips', `${h}.zip`)));
+
+	if (opts.ratings) {
+		// Upstream rebalances ratings; a new dated snapshot is written only when a hash changed.
+		await exportLeaderboards(playedHashes(0), true);
+	}
+
+	const summary = {
+		lastRun: started.toISOString(),
+		finished: new Date().toISOString(),
+		newBeatLeaderScores: newBL.map((s) => s.id),
+		newScoreSaberScores: newSS.map((it) => it.score.id),
+		ratingsRefreshed: opts.ratings,
+		mapsWithoutZip: noZip
+	};
+	updateState('sync', summary);
+	console.log('');
+	console.log(`Sync done: ${newBL.length} new BeatLeader scores, ${newSS.length} new ScoreSaber scores${opts.ratings ? ', ratings refreshed' : ''}.`);
+	if (noZip.length) console.log(`Maps without a zip (see beatsaver/missing.json; the game's CustomLevels are the manual fallback): ${noZip.join(' ')}`);
+	const failures = readState().failures;
+	if (failures && Object.keys(failures).length) console.warn(`Recorded failures (retried on the next run): ${JSON.stringify(failures)}`);
+	return 0;
+}
+
+function report(since: number, mapsSince: number): void {
+	const count = (...p: string[]) => listFiles(archivePath(...p)).length;
+	const day = (t: number) => (t ? new Date(t * 1000).toISOString().slice(0, 10) : 'all time');
+	const bl = archivedScores(since);
+	const blWithReplay = bl.filter((s) => s.replay);
+	const blReplayMissing = blWithReplay.filter((s) => !exists(archivePath('beatleader', 'replays', decodeURIComponent(new URL(s.replay!).pathname.split('/').pop()!))));
+	const statsMissing = bl.filter((s) => !exists(archivePath('beatleader', 'scorestats', `${s.id}.json`)));
+	const ss = PLAYER_IDS.map((id) => {
+		const items = archivedSSScores(id, since);
+		const withReplay = items.filter((it) => it.score.hasReplay);
+		const missing = withReplay.filter((it) => !exists(archivePath('scoresaber', id, 'replays', `${it.score.id}.dat`)));
+		return { id, items, withReplay, missing };
+	});
+	const allHashes = playedHashes(since);
+	const noLeaderboard = allHashes.filter((h) => listFiles(archivePath('beatleader', 'leaderboards', h)).length === 0);
+	const hashes = mapHashes(Math.max(since, mapsSince));
+	const noZip = hashes.filter((h) => !exists(archivePath('beatsaver', 'zips', `${h}.zip`)));
+	let missingJson: Record<string, unknown> = {};
+	try {
+		missingJson = JSON.parse(fs.readFileSync(archivePath('beatsaver', 'missing.json'), 'utf-8'));
+	} catch {}
+	const du = (dir: string): number => {
+		let total = 0;
+		const walk = (d: string) => {
+			for (const e of fs.readdirSync(d, { withFileTypes: true })) {
+				const p = path.join(d, e.name);
+				if (e.isDirectory()) walk(p);
+				else total += fs.statSync(p).size;
+			}
+		};
+		if (fs.existsSync(dir)) walk(dir);
+		return total;
+	};
+
+	console.log(`Archive: ${ARCHIVE_DIR}`);
+	console.log(`Window:  scores since ${day(since)}, BeatSaver maps since ${day(mapsSince)}`);
+	console.log('');
+	console.log(`BeatLeader scores        ${bl.length} in window (${count('beatleader', 'scores')} archived total, both accounts)`);
+	console.log(`BeatLeader replays       ${blWithReplay.length - blReplayMissing.length}/${blWithReplay.length}`);
+	console.log(`BeatLeader score stats   ${bl.length - statsMissing.length}/${bl.length}`);
+	for (const s of ss) {
+		console.log(`ScoreSaber ${s.id}: ${s.items.length} scores, replays ${s.withReplay.length - s.missing.length}/${s.withReplay.length}`);
+	}
+	console.log(`Played hashes            ${allHashes.length}`);
+	console.log(`  BL leaderboard snapshot ${allHashes.length - noLeaderboard.length}/${allHashes.length}`);
+	console.log(`  BeatSaver zip           ${hashes.length - noZip.length}/${hashes.length} in maps window (missing.json lists ${Object.keys(missingJson).length})`);
+	console.log(`Archive size             ${(du(ARCHIVE_DIR) / 1e9).toFixed(2)} GB`);
+	const failures = readState().failures;
+	if (failures && Object.keys(failures).length) console.log(`Failures from last runs: ${JSON.stringify(failures)}`);
+}
+
+function option(args: string[], name: string): string | undefined {
+	const i = args.indexOf(name);
+	return i >= 0 ? args[i + 1] : args.find((a) => a.startsWith(`${name}=`))?.slice(name.length + 1);
+}
+
+async function main(): Promise<number> {
+	loadDotEnv();
+	const args = process.argv.slice(2);
+	const [cmd, sub] = args;
+
+	if (cmd === 'auth' && sub === 'login') {
+		const store = await login();
+		console.log(`\nLogged in as ${store.player_name} (${store.player_id}). Tokens saved to ${TOKENS_FILE}`);
+		console.log(`Refresh token valid until about ${fmt(store.refresh_expires_at)}`);
+		return (await verify()) ? 0 : 1;
+	}
+	if (cmd === 'auth' && sub === 'refresh') {
+		const store = await refresh();
+		console.log(`Refreshed. Access token until ${fmt(store.access_expires_at)}; refresh until about ${fmt(store.refresh_expires_at)}`);
+		return 0;
+	}
+	if (cmd === 'auth' && sub === 'status') {
+		const store = readTokens();
+		if (!store) {
+			console.log(`No tokens at ${TOKENS_FILE}. Run \`auth login\`.`);
+			return 1;
+		}
+		console.log(`Tokens:  ${TOKENS_FILE}`);
+		console.log(`Player:  ${store.player_name} (${store.player_id})`);
+		console.log(`Access:  ${fmt(store.access_expires_at)}`);
+		console.log(`Refresh: ${fmt(store.refresh_expires_at)} (estimated)`);
+		return (await verify()) ? 0 : 1;
+	}
+
+	const since = parseSince(option(args, '--since'));
+	const mapsSince = Math.max(since, parseSince(option(args, '--maps-since'), DEFAULT_MAPS_SINCE));
+	if (cmd === 'export') {
+		console.log(`Archive ${ARCHIVE_DIR}, since ${since ? new Date(since * 1000).toISOString() : 'all time'}\n`);
+		switch (sub) {
+			case 'bl':
+				await exportBL(since);
+				return 0;
+			case 'ss':
+				await exportSS(since);
+				return 0;
+			case 'ss-replays':
+				// Legacy first: its replays are mostly already on disk from ss-scraper.
+				for (const id of [...PLAYER_IDS].reverse()) await exportSSReplays(id, since);
+				return 0;
+			case 'maps':
+				await exportMaps(mapHashes(mapsSince));
+				return 0;
+			case 'all':
+				await exportBL(since);
+				await exportSS(since);
+				await exportMaps(mapHashes(mapsSince));
+				console.log('');
+				report(since, mapsSince);
+				return 0;
+		}
+	}
+	if (cmd === 'sync') {
+		// Sundays in the machine's timezone (America/Los_Angeles on incineroar).
+		const ratings = args.includes('--ratings') || new Date().getDay() === 0;
+		return sync({ ratings, ssReplays: !args.includes('--no-ss-replays') });
+	}
+	if (cmd === 'report') {
+		report(since, mapsSince);
+		return 0;
+	}
+
+	console.error('Usage: node scripts/archive/cli.ts <auth login|refresh|status | export bl|ss|ss-replays|maps|all | report | sync [--ratings] [--no-ss-replays]> [--since YYYY-MM-DD|all] [--maps-since YYYY-MM-DD|all]');
+	return 2;
+}
+
+main().then(
+	(code) => process.exit(code),
+	(err) => {
+		console.error(err instanceof Error ? err.message : err);
+		process.exit(1);
+	}
+);

scripts/archive/config.ts

@@ -0,0 +1,76 @@
+// Shared configuration for the plebleader archive CLI.
+// Runs directly under Node >= 22 (type stripping), no build step.
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+export const USER_AGENT = 'plebsaber-archive/0.1';
+export const BL_API = 'https://api.beatleader.com';
+export const MAIN_PLAYER_ID = '76561199407393962';
+export const LEGACY_PLAYER_ID = '76561197962107242';
+
+export const PLAYER_IDS = [MAIN_PLAYER_ID, LEGACY_PLAYER_ID];
+
+/** Scores, replays, score stats and profile data: all history by default. */
+export const DEFAULT_SINCE = 'all';
+/**
+ * BeatSaver map files (JSON + zip) only for maps played in the last two years. Older maps can
+ * be fetched later with --maps-since all (existing files are skipped).
+ */
+export const DEFAULT_MAPS_SINCE = '2024-11-08';
+
+/** Where ss-scraper already downloaded ScoreSaber replays (<playerId>/<scoreId>_<name>.dat). */
+export const SS_SCRAPER_REPLAYS = process.env.SS_SCRAPER_REPLAYS ?? path.join(process.env.HOME ?? '', 'ops/beatsaber/ss-scraper/replays');
+
+/** Parse --since: YYYY-MM-DD (UTC), unix seconds, or "all". */
+export function parseSince(value: string | undefined, fallback = DEFAULT_SINCE): number {
+	const v = value ?? fallback;
+	if (v === 'all') return 0;
+	if (/^\d+$/.test(v)) return Number(v);
+	const ms = Date.parse(/^\d{4}-\d{2}-\d{2}$/.test(v) ? `${v}T00:00:00Z` : v);
+	if (Number.isNaN(ms)) throw new Error(`Invalid --since: ${v}`);
+	return Math.floor(ms / 1000);
+}
+
+/**
+ * Root of the plebleader state (archive/, auth/, db/, ...).
+ * Production: /var/lib/plebleader. Local default: ./.data/plebleader (gitignored).
+ */
+export const ROOT = path.resolve(process.env.PLEBLEADER_DIR ?? path.join(process.cwd(), '.data', 'plebleader'));
+export const ARCHIVE_DIR = path.join(ROOT, 'archive');
+export const AUTH_DIR = path.join(ROOT, 'auth');
+export const TOKENS_FILE = path.join(AUTH_DIR, 'beatleader_tokens.json');
+
+/** Load KEY=VALUE (optionally `export KEY=VALUE`) lines from .env without overriding the real environment. */
+export function loadDotEnv(file = path.join(process.cwd(), '.env')): void {
+	if (!fs.existsSync(file)) return;
+	for (const line of fs.readFileSync(file, 'utf-8').split('\n')) {
+		const m = line.match(/^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/);
+		if (!m) continue;
+		const [, key, raw] = m;
+		const value = raw.replace(/^(['"])(.*)\1$/, '$2');
+		if (process.env[key] === undefined) process.env[key] = value;
+	}
+}
+
+export function clientCredentials(): { clientId: string; clientSecret: string } {
+	const clientId = process.env.BL_CLIENT_ID;
+	const clientSecret = process.env.BL_CLIENT_SECRET;
+	if (!clientId || !clientSecret) {
+		throw new Error('BL_CLIENT_ID and BL_CLIENT_SECRET must be set (environment or .env).');
+	}
+	return { clientId, clientSecret };
+}
+
+/** Write a file atomically: temp file in the same directory, fsync, rename. */
+export function writeFileAtomic(file: string, data: string | Uint8Array, mode = 0o644): void {
+	fs.mkdirSync(path.dirname(file), { recursive: true });
+	const tmp = `${file}.tmp-${process.pid}-${Date.now()}`;
+	const fd = fs.openSync(tmp, 'w', mode);
+	try {
+		fs.writeSync(fd, typeof data === 'string' ? Buffer.from(data, 'utf-8') : data);
+		fs.fsyncSync(fd);
+	} finally {
+		fs.closeSync(fd);
+	}
+	fs.renameSync(tmp, file);
+}

scripts/archive/http.ts

@@ -0,0 +1,107 @@
+// Rate-limited GET with 429 / 5xx / network retries. Returns raw bytes so callers can
+// store responses byte-for-byte.
+import { USER_AGENT } from './config.ts';
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+/** Enforces a minimum interval between request starts. */
+export class Limiter {
+	private next = 0;
+	private readonly minIntervalMs: number;
+	constructor(minIntervalMs: number) {
+		this.minIntervalMs = minIntervalMs;
+	}
+	async wait(): Promise<void> {
+		const now = Date.now();
+		const at = Math.max(now, this.next);
+		this.next = at + this.minIntervalMs;
+		if (at > now) await sleep(at - now);
+	}
+	/** Push the next slot out, e.g. after a 429. */
+	backoff(ms: number): void {
+		this.next = Math.max(this.next, Date.now() + ms);
+	}
+}
+
+export const limiters = {
+	beatleader: new Limiter(250), // ~4 req/s, well under 50 per 10 s
+	blCdn: new Limiter(250),
+	scoresaber: new Limiter(1000),
+	// /api/v2/scores/{id}/replay sends x-ratelimit-limit-long: 360 per hour (checked 2026-11-08;
+	// short 20 and medium 60/min don't bind). 10.5 s ≈ 343/h. get() also pauses at remaining 0.
+	scoresaberReplay: new Limiter(10_500),
+	beatsaver: new Limiter(500), // ~2 req/s
+	beatsaverCdn: new Limiter(250)
+};
+
+export type GetResult = { status: number; body: Buffer; headers: Headers; url: string };
+
+export class HttpError extends Error {
+	readonly status: number;
+	constructor(status: number, message: string) {
+		super(message);
+		this.status = status;
+	}
+}
+
+/**
+ * GET with retries. Resolves with the final response for 2xx and for statuses listed in
+ * `accept` (e.g. 404); throws HttpError for anything else once retries are exhausted.
+ */
+export async function get(
+	url: string,
+	opts: { limiter: Limiter; headers?: Record<string, string>; accept?: number[]; retries?: number; timeoutMs?: number }
+): Promise<GetResult> {
+	const retries = opts.retries ?? 6;
+	let backoffMs = 2000;
+	for (let attempt = 0; ; attempt++) {
+		await opts.limiter.wait();
+		let res: Response;
+		try {
+			res = await fetch(url, {
+				headers: { 'User-Agent': USER_AGENT, ...opts.headers },
+				signal: AbortSignal.timeout(opts.timeoutMs ?? 120_000)
+			});
+		} catch (err) {
+			if (attempt >= retries) throw err;
+			console.warn(`  network error on ${shortUrl(url)}: ${(err as Error).message}; retrying in ${backoffMs / 1000}s`);
+			await sleep(backoffMs);
+			backoffMs = Math.min(backoffMs * 2, 300_000);
+			continue;
+		}
+		honorRateLimitHeaders(res.headers, opts.limiter, url);
+		if (res.ok || opts.accept?.includes(res.status)) {
+			return { status: res.status, body: Buffer.from(await res.arrayBuffer()), headers: res.headers, url: res.url };
+		}
+		const text = (await res.text().catch(() => '')).slice(0, 200);
+		const retryable = res.status === 429 || res.status >= 500;
+		if (!retryable || attempt >= retries) {
+			throw new HttpError(res.status, `GET ${shortUrl(url)}: HTTP ${res.status} ${text}`);
+		}
+		const retryAfter = Number(res.headers.get('retry-after'));
+		const wait = Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter * 1000 : backoffMs;
+		console.warn(`  HTTP ${res.status} on ${shortUrl(url)}; waiting ${Math.round(wait / 1000)}s`);
+		opts.limiter.backoff(wait);
+		backoffMs = Math.min(backoffMs * 2, 300_000);
+	}
+}
+
+/**
+ * ScoreSaber-style headers: x-ratelimit-remaining-{short,medium,long} and
+ * x-ratelimit-reset-{...} (seconds until that window resets). When any window is exhausted,
+ * hold the limiter until it resets instead of running into a 429.
+ */
+function honorRateLimitHeaders(headers: Headers, limiter: Limiter, url: string): void {
+	for (const w of ['short', 'medium', 'long']) {
+		const remaining = headers.get(`x-ratelimit-remaining-${w}`);
+		const reset = Number(headers.get(`x-ratelimit-reset-${w}`));
+		if (remaining !== null && Number(remaining) <= 0 && Number.isFinite(reset) && reset > 0) {
+			console.warn(`  rate limit window "${w}" exhausted on ${shortUrl(url)}; pausing ${reset}s`);
+			limiter.backoff((reset + 1) * 1000);
+		}
+	}
+}
+
+function shortUrl(url: string): string {
+	return url.split('?')[0];
+}

scripts/archive/scoresaber.ts

@@ -0,0 +1,115 @@
+// ScoreSaber exporter: player snapshot, scores (score + leaderboard items), replays.
+// Public API, no auth. Replays are paced at ~10.5 s (360/hour limit) and run as their own command.
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { SS_SCRAPER_REPLAYS } from './config.ts';
+import { get, limiters } from './http.ts';
+import { archivePath, eachWithProgress, exists, listFiles, readJson, save, saveSnapshotIfChanged, updateState } from './store.ts';
+
+const SS_API = 'https://scoresaber.com/api';
+const ss = (playerId: string, ...p: string[]) => archivePath('scoresaber', playerId, ...p);
+
+export type SSItem = {
+	score: { id: number; timeSet: string; hasReplay: boolean };
+	leaderboard: { songHash: string };
+};
+
+export function archivedSSScores(playerId: string, sinceUnix = 0): SSItem[] {
+	const dir = ss(playerId, 'scores');
+	return listFiles(dir, '.json')
+		.map((f) => readJson<SSItem>(path.join(dir, f)))
+		.filter((it) => Date.parse(it.score.timeSet) / 1000 >= sinceUnix);
+}
+
+export async function exportSSPlayer(playerId: string): Promise<void> {
+	const r = await get(`${SS_API}/player/${playerId}/full`, { limiter: limiters.scoresaber });
+	console.log(`ScoreSaber player ${playerId}: ${saveSnapshotIfChanged(ss(playerId, 'player'), r.body) ? 'saved' : 'unchanged'}`);
+}
+
+/** Page `sort=recent` until scores are older than `sinceUnix`. Returns the new items. */
+export async function exportSSScores(playerId: string, sinceUnix: number): Promise<SSItem[]> {
+	console.log(`ScoreSaber scores for ${playerId} since ${new Date(sinceUnix * 1000).toISOString()}`);
+	const added: SSItem[] = [];
+	let seen = 0;
+	for (let page = 1; ; page++) {
+		const r = await get(`${SS_API}/player/${playerId}/scores?limit=100&page=${page}&sort=recent`, { limiter: limiters.scoresaber });
+		const json = JSON.parse(r.body.toString('utf-8')) as { playerScores: SSItem[]; metadata: { total: number } };
+		let reachedCutoff = false;
+		for (const item of json.playerScores) {
+			if (Date.parse(item.score.timeSet) / 1000 < sinceUnix) {
+				reachedCutoff = true;
+				break;
+			}
+			seen++;
+			const file = ss(playerId, 'scores', `${item.score.id}.json`);
+			if (exists(file)) continue;
+			save(file, JSON.stringify(item));
+			added.push(item);
+		}
+		console.log(`  page ${page}: ${seen} in window, ${added.length} new (total on server ${json.metadata.total})`);
+		if (reachedCutoff || json.playerScores.length < 100) break;
+	}
+	updateState(`scoresaber:${playerId}`, { scoresSince: sinceUnix, lastScoresRun: new Date().toISOString() });
+	return added;
+}
+
+/** Newest archived ScoreSaber score time for a player (unix seconds), 0 when there is none. */
+export function newestSSScoreTime(playerId: string): number {
+	return archivedSSScores(playerId).reduce((max, it) => Math.max(max, Math.floor(Date.parse(it.score.timeSet) / 1000)), 0);
+}
+
+/** Download replays for archived scores with hasReplay, oldest first, ~10.5 s apart. */
+export async function exportSSReplays(playerId: string, sinceUnix: number): Promise<void> {
+	const items = archivedSSScores(playerId, sinceUnix)
+		.filter((it) => it.score.hasReplay)
+		.sort((a, b) => Date.parse(a.score.timeSet) - Date.parse(b.score.timeSet));
+	seedFromScraper(playerId, items);
+	const todo = items.filter((it) => !exists(ss(playerId, 'replays', `${it.score.id}.dat`)));
+	console.log(`ScoreSaber replays for ${playerId}: ${items.length} with hasReplay, ${todo.length} to download (~${Math.ceil((todo.length * 10.5) / 60)} min)`);
+	const res = await eachWithProgress(
+		'ss-replays',
+		todo,
+		async (it) => {
+			const r = await get(`${SS_API}/v2/scores/${it.score.id}/replay`, {
+				limiter: limiters.scoresaberReplay,
+				accept: [404],
+				timeoutMs: 300_000
+			});
+			if (r.status === 404) return 'missing';
+			save(ss(playerId, 'replays', `${it.score.id}.dat`), r.body);
+			return 'done';
+		},
+		10
+	);
+	if (res.missing || res.failed.length) {
+		updateState('failures', { [`scoresaber:${playerId}:replays`]: res.failed.map((f) => f.item.score.id) });
+	}
+}
+
+/** Copy replays ss-scraper already downloaded (<scoreId>_<name>.dat) instead of fetching them again. */
+function seedFromScraper(playerId: string, items: SSItem[]): void {
+	const dir = path.join(SS_SCRAPER_REPLAYS, playerId);
+	const byId = new Map<string, string>();
+	for (const f of listFiles(dir, '.dat')) byId.set(f.split('_')[0], f);
+	let copied = 0;
+	for (const it of items) {
+		const src = byId.get(String(it.score.id));
+		const dest = ss(playerId, 'replays', `${it.score.id}.dat`);
+		if (!src || exists(dest)) continue;
+		save(dest, fs.readFileSync(path.join(dir, src)));
+		copied++;
+	}
+	if (copied) console.log(`  copied ${copied} replays from ${dir}`);
+}
+
+/**
+ * The map hash BeatLeader and the archive use for a ScoreSaber item: lowercase, and for official
+ * levels the level id without ScoreSaber's "ost_" prefix (BeatLeader: "beatsaber", SS: "ost_BeatSaber").
+ */
+export function ssMapHash(it: SSItem): string {
+	return it.leaderboard.songHash.toLowerCase().replace(/^ost_/, '');
+}
+
+export function ssHashes(playerId: string, sinceUnix = 0): string[] {
+	return [...new Set(archivedSSScores(playerId, sinceUnix).map(ssMapHash))].sort();
+}

scripts/archive/store.ts

@@ -0,0 +1,80 @@
+// Archive filesystem helpers. The archive is append-only: files are written once
+// (atomically) and never rewritten, except state.json.
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { ARCHIVE_DIR, writeFileAtomic } from './config.ts';
+
+export const archivePath = (...parts: string[]) => path.join(ARCHIVE_DIR, ...parts);
+
+export const exists = (file: string) => fs.existsSync(file);
+
+export function save(file: string, data: string | Uint8Array): void {
+	writeFileAtomic(file, data);
+}
+
+/** Today's date (UTC) as YYYY-MM-DD, used for dated snapshot filenames. */
+export const today = () => new Date().toISOString().slice(0, 10);
+
+export function listFiles(dir: string, ext?: string): string[] {
+	try {
+		return fs.readdirSync(dir).filter((f) => !f.includes('.tmp-') && (!ext || f.endsWith(ext)));
+	} catch {
+		return [];
+	}
+}
+
+export function readJson<T>(file: string): T {
+	return JSON.parse(fs.readFileSync(file, 'utf-8')) as T;
+}
+
+/**
+ * Save a dated snapshot in `dir` only when the content differs from the newest existing one.
+ * Returns true if a new snapshot was written.
+ */
+export function saveSnapshotIfChanged(dir: string, body: Buffer): boolean {
+	const latest = listFiles(dir, '.json').sort().at(-1);
+	if (latest && fs.readFileSync(path.join(dir, latest)).equals(body)) return false;
+	save(path.join(dir, `${today()}.json`), body);
+	return true;
+}
+
+type State = Record<string, Record<string, unknown>>;
+const STATE_FILE = () => archivePath('state.json');
+
+export function readState(): State {
+	try {
+		return readJson<State>(STATE_FILE());
+	} catch {
+		return {};
+	}
+}
+
+export function updateState(section: string, patch: Record<string, unknown>): void {
+	const state = readState();
+	state[section] = { ...state[section], ...patch };
+	writeFileAtomic(STATE_FILE(), JSON.stringify(state, null, 2) + '\n');
+}
+
+/** Run `fn` over items with a progress line every `every` items. Errors are collected, not thrown. */
+export async function eachWithProgress<T>(
+	label: string,
+	items: T[],
+	fn: (item: T) => Promise<'done' | 'skipped' | 'missing'>,
+	every = 50
+): Promise<{ done: number; skipped: number; missing: number; failed: { item: T; error: string }[] }> {
+	const out = { done: 0, skipped: 0, missing: 0, failed: [] as { item: T; error: string }[] };
+	let i = 0;
+	for (const item of items) {
+		i++;
+		try {
+			out[await fn(item)]++;
+		} catch (err) {
+			out.failed.push({ item, error: (err as Error).message });
+			console.warn(`  ${label}: ${(err as Error).message}`);
+		}
+		if (i % every === 0 || i === items.length) {
+			console.log(`  ${label}: ${i}/${items.length} (new ${out.done}, existing ${out.skipped}, missing ${out.missing}, failed ${out.failed.length})`);
+		}
+	}
+	return out;
+}