Add the plebleader daily sync and package the archive CLI

`sync` refreshes the token chain first, then fetches new BeatLeader and
ScoreSaber scores (with a day of overlap), their replays, score stats,
leaderboards and map zips, and re-snapshots every played hash on Sundays.
The flake's plebleader-archive package runs the CLI with Node's type
stripping.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

flake.nix

@@ -44,12 +44,35 @@
           mainProgram = "plebsaber-stream";
         };
       };
+    # The plebleader archive CLI (scripts/archive, docs/plan.md): plain
+    # TypeScript with only Node built-ins, run through Node's type stripping.
+    mkArchiveCli = pkgs:
+      pkgs.stdenvNoCC.mkDerivation {
+        pname = "plebleader-archive";
+        version = "0.1.0";
+        src = ./scripts/archive;
+        nativeBuildInputs = [ pkgs.makeWrapper ];
+        installPhase = ''
+          runHook preInstall
+          mkdir -p $out/lib/plebleader-archive
+          cp *.ts $out/lib/plebleader-archive/
+          makeWrapper ${pkgs.nodejs_22}/bin/node $out/bin/plebleader-archive \
+            --add-flags "--experimental-strip-types --disable-warning=ExperimentalWarning" \
+            --add-flags "$out/lib/plebleader-archive/cli.ts"
+          runHook postInstall
+        '';
+        meta = {
+          description = "Archive exporter and daily sync for the personal BeatLeader mirror";
+          mainProgram = "plebleader-archive";
+        };
+      };
   in rec {
     packages = forAllSystems (system: let
       pkgs = import nixpkgs { inherit system; };
       pkg = mkPackage pkgs;
     in {
       plebsaber-stream = pkg;
+      plebleader-archive = mkArchiveCli pkgs;
       default = pkg;
     });
 

scripts/archive/beatleader.ts

@@ -3,11 +3,11 @@
 import * as path from 'node:path';
 import { BL_API } from './config.ts';
 import { get, limiters } from './http.ts';
-import { archivePath, eachWithProgress, exists, listFiles, readJson, save, saveSnapshotIfChanged, today, updateState } from './store.ts';
+import { archivePath, eachWithProgress, exists, listFiles, readJson, readState, save, saveSnapshotIfChanged, today, updateState } from './store.ts';
 
 const bl = (...p: string[]) => archivePath('beatleader', ...p);
 
-type BLScore = { id: number; timepost: number; replay?: string | null; leaderboard: { song: { hash: string } } };
+export type BLScore = { id: number; playerId: string; timepost: number; replay?: string | null; leaderboard: { song: { hash: string } } };
 
 async function blGet(pathAndQuery: string, accept?: number[]) {
 	return get(`${BL_API}${pathAndQuery}`, { limiter: limiters.beatleader, accept });
@@ -39,12 +39,12 @@ export async function exportProfile(playerId: string): Promise<void> {
 	}
 }
 
-/** Page through best scores newest-first from `sinceUnix`, saving each score object. */
+/** Page through best scores newest-first from `sinceUnix`, saving each score object. Returns the new ones. */
-export async function exportScores(playerId: string, sinceUnix: number): Promise<void> {
+export async function exportScores(playerId: string, sinceUnix: number): Promise<BLScore[]> {
 	console.log(`BeatLeader scores for ${playerId} since ${new Date(sinceUnix * 1000).toISOString()}`);
-	let added = 0;
+	const added: BLScore[] = [];
 	let seen = 0;
-	let newest = 0;
+	let newest = Number(readState()[`beatleader:${playerId}`]?.newestTimepost ?? 0);
 	for (let page = 1; ; page++) {
 		const qs = `count=100&page=${page}&sortBy=date&order=desc&includeIO=true&time_from=${sinceUnix}`;
 		const res = await blGet(`/player/${playerId}/scores?${qs}`);
@@ -56,12 +56,18 @@ export async function exportScores(playerId: string, sinceUnix: number): Promise
 			if (exists(file)) continue;
 			// The page is parsed and each element re-serialized: same content, no reshaping.
 			save(file, JSON.stringify(score));
-			added++;
+			added.push(score);
 		}
-		console.log(`  page ${page}: ${seen}/${json.metadata.total} seen, ${added} new`);
+		console.log(`  page ${page}: ${seen}/${json.metadata.total} seen, ${added.length} new`);
 		if (json.data.length < 100 || seen >= json.metadata.total) break;
 	}
 	updateState(`beatleader:${playerId}`, { scoresSince: sinceUnix, newestTimepost: newest, lastScoresRun: new Date().toISOString() });
+	return added;
+}
+
+/** Newest archived score time for a player (unix seconds), 0 when there is none. */
+export function newestScoreTime(playerId: string): number {
+	return archivedScores().reduce((max, s) => (s.playerId === playerId ? Math.max(max, s.timepost) : max), 0);
 }
 
 export async function exportReplays(scores: BLScore[]): Promise<void> {

scripts/archive/cli.ts

@@ -9,6 +9,10 @@
 //                                   plus all legacy ScoreSaber maps)
 //   export all         [--since D]  bl, ss, maps (not ss-replays)
 //   report             [--since D]  counts and gaps in the archive
+//   sync    [--ratings] [--no-ss-replays]
+//                                   daily incremental run (phase 6): token refresh, new BL and SS
+//                                   scores with their replays, stats, leaderboards and maps.
+//                                   --ratings re-snapshots every played hash (default on Sundays)
 //
 // --since: YYYY-MM-DD, unix seconds, or "all" (default: all). --maps-since defaults to the two-year
 // window (DEFAULT_MAPS_SINCE). Every command skips files that already exist, so reruns only fill gaps.
@@ -16,10 +20,10 @@ import * as fs from 'node:fs';
 import * as path from 'node:path';
 import { ARCHIVE_DIR, BL_API, DEFAULT_MAPS_SINCE, LEGACY_PLAYER_ID, MAIN_PLAYER_ID, PLAYER_IDS, TOKENS_FILE, USER_AGENT, loadDotEnv, parseSince } from './config.ts';
 import { getAccessToken, login, readTokens, refresh } from './auth.ts';
-import { archivedScores, blHashes, exportLeaderboards, exportProfile, exportReplays, exportScoreStats, exportScores } from './beatleader.ts';
+import { archivedScores, blHashes, exportLeaderboards, exportProfile, exportReplays, exportScoreStats, exportScores, newestScoreTime } from './beatleader.ts';
-import { archivedSSScores, exportSSPlayer, exportSSReplays, exportSSScores, ssHashes } from './scoresaber.ts';
+import { archivedSSScores, exportSSPlayer, exportSSReplays, exportSSScores, newestSSScoreTime, ssHashes } from './scoresaber.ts';
 import { exportMaps } from './beatsaver.ts';
-import { archivePath, exists, listFiles, readState } from './store.ts';
+import { archivePath, exists, listFiles, readState, updateState } from './store.ts';
 
 function fmt(ms: number): string {
 	const d = new Date(ms);
@@ -106,6 +110,64 @@ async function exportSS(since: number): Promise<void> {
 	await exportLeaderboards([...new Set(PLAYER_IDS.flatMap((id) => ssHashes(id, since)))].sort());
 }
 
+/** Overlap of each incremental window, so scores posted late or with a skewed clock aren't missed. */
+const SYNC_OVERLAP_S = 24 * 3600;
+
+/**
+ * Phase 6: the daily run. Main account only; the legacy accounts are frozen. Every step skips
+ * what's already archived, so a rerun (or a run after a missed day) just fills the gap.
+ */
+async function sync(opts: { ratings: boolean; ssReplays: boolean }): Promise<number> {
+	const started = new Date();
+	// Token upkeep first: persist the rotated refresh token before anything else, and fail loudly.
+	const tokens = await refresh();
+	console.log(`Token refreshed; refresh chain valid until about ${fmt(tokens.refresh_expires_at)}\n`);
+
+	await exportProfile(MAIN_PLAYER_ID);
+	const blSince = Math.max(0, newestScoreTime(MAIN_PLAYER_ID) - SYNC_OVERLAP_S);
+	const newBL = await exportScores(MAIN_PLAYER_ID, blSince);
+	// The whole window, not just the new scores, so a replay or stats file that failed yesterday is retried.
+	const blWindow = archivedScores(blSince).filter((s) => s.playerId === MAIN_PLAYER_ID);
+	await exportReplays(blWindow);
+	await exportScoreStats(blWindow);
+	// A new score can change a leaderboard (status, ratings), so re-snapshot its hash.
+	const newBLHashes = [...new Set(newBL.map((s) => s.leaderboard.song.hash.toLowerCase()))].sort();
+	await exportLeaderboards(newBLHashes, true);
+
+	await exportSSPlayer(MAIN_PLAYER_ID);
+	const ssSince = Math.max(0, newestSSScoreTime(MAIN_PLAYER_ID) - SYNC_OVERLAP_S);
+	const newSS = await exportSSScores(MAIN_PLAYER_ID, ssSince);
+	const newSSHashes = [...new Set(newSS.map((it) => it.leaderboard.songHash.toLowerCase()))].sort();
+	await exportLeaderboards(newSSHashes.filter((h) => !newBLHashes.includes(h)));
+	if (opts.ssReplays) await exportSSReplays(MAIN_PLAYER_ID, ssSince);
+
+	// Maps for everything played in either window; existing zips are skipped.
+	const windowHashes = [...new Set([...blWindow.map((s) => s.leaderboard.song.hash.toLowerCase()), ...ssHashes(MAIN_PLAYER_ID, ssSince)])].sort();
+	await exportMaps(windowHashes);
+	const noZip = windowHashes.filter((h) => !exists(archivePath('beatsaver', 'zips', `${h}.zip`)));
+
+	if (opts.ratings) {
+		// Upstream rebalances ratings; a new dated snapshot is written only when a hash changed.
+		await exportLeaderboards(playedHashes(0), true);
+	}
+
+	const summary = {
+		lastRun: started.toISOString(),
+		finished: new Date().toISOString(),
+		newBeatLeaderScores: newBL.map((s) => s.id),
+		newScoreSaberScores: newSS.map((it) => it.score.id),
+		ratingsRefreshed: opts.ratings,
+		mapsWithoutZip: noZip
+	};
+	updateState('sync', summary);
+	console.log('');
+	console.log(`Sync done: ${newBL.length} new BeatLeader scores, ${newSS.length} new ScoreSaber scores${opts.ratings ? ', ratings refreshed' : ''}.`);
+	if (noZip.length) console.log(`Maps without a zip (see beatsaver/missing.json; the game's CustomLevels are the manual fallback): ${noZip.join(' ')}`);
+	const failures = readState().failures;
+	if (failures && Object.keys(failures).length) console.warn(`Recorded failures (retried on the next run): ${JSON.stringify(failures)}`);
+	return 0;
+}
+
 function report(since: number, mapsSince: number): void {
 	const count = (...p: string[]) => listFiles(archivePath(...p)).length;
 	const day = (t: number) => (t ? new Date(t * 1000).toISOString().slice(0, 10) : 'all time');
@@ -218,12 +280,17 @@ async function main(): Promise<number> {
 				return 0;
 		}
 	}
+	if (cmd === 'sync') {
+		// Sundays in the machine's timezone (America/Los_Angeles on incineroar).
+		const ratings = args.includes('--ratings') || new Date().getDay() === 0;
+		return sync({ ratings, ssReplays: !args.includes('--no-ss-replays') });
+	}
 	if (cmd === 'report') {
 		report(since, mapsSince);
 		return 0;
 	}
 
-	console.error('Usage: node scripts/archive/cli.ts <auth login|refresh|status | export bl|ss|ss-replays|maps|all | report> [--since YYYY-MM-DD|all] [--maps-since YYYY-MM-DD|all]');
+	console.error('Usage: node scripts/archive/cli.ts <auth login|refresh|status | export bl|ss|ss-replays|maps|all | report | sync [--ratings] [--no-ss-replays]> [--since YYYY-MM-DD|all] [--maps-since YYYY-MM-DD|all]');
 	return 2;
 }
 

scripts/archive/scoresaber.ts

@@ -26,10 +26,10 @@ export async function exportSSPlayer(playerId: string): Promise<void> {
 	console.log(`ScoreSaber player ${playerId}: ${saveSnapshotIfChanged(ss(playerId, 'player'), r.body) ? 'saved' : 'unchanged'}`);
 }
 
-/** Page `sort=recent` until scores are older than `sinceUnix`. */
+/** Page `sort=recent` until scores are older than `sinceUnix`. Returns the new items. */
-export async function exportSSScores(playerId: string, sinceUnix: number): Promise<void> {
+export async function exportSSScores(playerId: string, sinceUnix: number): Promise<SSItem[]> {
 	console.log(`ScoreSaber scores for ${playerId} since ${new Date(sinceUnix * 1000).toISOString()}`);
-	let added = 0;
+	const added: SSItem[] = [];
 	let seen = 0;
 	for (let page = 1; ; page++) {
 		const r = await get(`${SS_API}/player/${playerId}/scores?limit=100&page=${page}&sort=recent`, { limiter: limiters.scoresaber });
@@ -44,12 +44,18 @@ export async function exportSSScores(playerId: string, sinceUnix: number): Promi
 			const file = ss(playerId, 'scores', `${item.score.id}.json`);
 			if (exists(file)) continue;
 			save(file, JSON.stringify(item));
-			added++;
+			added.push(item);
 		}
-		console.log(`  page ${page}: ${seen} in window, ${added} new (total on server ${json.metadata.total})`);
+		console.log(`  page ${page}: ${seen} in window, ${added.length} new (total on server ${json.metadata.total})`);
 		if (reachedCutoff || json.playerScores.length < 100) break;
 	}
 	updateState(`scoresaber:${playerId}`, { scoresSince: sinceUnix, lastScoresRun: new Date().toISOString() });
+	return added;
+}
+
+/** Newest archived ScoreSaber score time for a player (unix seconds), 0 when there is none. */
+export function newestSSScoreTime(playerId: string): number {
+	return archivedSSScores(playerId).reduce((max, it) => Math.max(max, Math.floor(Date.parse(it.score.timeSet) / 1000)), 0);
 }
 
 /** Download replays for archived scores with hasReplay, oldest first, ~10.5 s apart. */