pleb/plebsaber.stream
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add the plebleader daily sync and package the archive CLI

Browse Source 
`sync` refreshes the token chain first, then fetches new BeatLeader and
ScoreSaber scores (with a day of overlap), their replays, score stats,
leaderboards and map zips, and re-snapshots every played hash on Sundays.
The flake's plebleader-archive package runs the CLI with Node's type
stripping.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
pleb
2026-11-09 07:55:58 -08:00
co-authored by Claude Opus 5.5
parent 51c1e6d3d1
commit d182fd808c
4 changed files with 119 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/archive/cli.ts
+71 -4
View File
@@ -9,6 +9,10 @@
// plus all legacy ScoreSaber maps)
// export all [--since D] bl, ss, maps (not ss-replays)
// report [--since D] counts and gaps in the archive
// sync [--ratings] [--no-ss-replays]
// daily incremental run (phase 6): token refresh, new BL and SS
// scores with their replays, stats, leaderboards and maps.
// --ratings re-snapshots every played hash (default on Sundays)
//
// --since: YYYY-MM-DD, unix seconds, or "all" (default: all). --maps-since defaults to the two-year
// window (DEFAULT_MAPS_SINCE). Every command skips files that already exist, so reruns only fill gaps.
@@ -16,10 +20,10 @@ import * as fs from 'node:fs';
import * as path from 'node:path';
import { ARCHIVE_DIR, BL_API, DEFAULT_MAPS_SINCE, LEGACY_PLAYER_ID, MAIN_PLAYER_ID, PLAYER_IDS, TOKENS_FILE, USER_AGENT, loadDotEnv, parseSince } from './config.ts';
import { getAccessToken, login, readTokens, refresh } from './auth.ts';
import { archivedScores, blHashes, exportLeaderboards, exportProfile, exportReplays, exportScoreStats, exportScores } from './beatleader.ts';
import { archivedSSScores, exportSSPlayer, exportSSReplays, exportSSScores, ssHashes } from './scoresaber.ts';
import { archivedScores, blHashes, exportLeaderboards, exportProfile, exportReplays, exportScoreStats, exportScores, newestScoreTime } from './beatleader.ts';
import { archivedSSScores, exportSSPlayer, exportSSReplays, exportSSScores, newestSSScoreTime, ssHashes } from './scoresaber.ts';
import { exportMaps } from './beatsaver.ts';
import { archivePath, exists, listFiles, readState } from './store.ts';
import { archivePath, exists, listFiles, readState, updateState } from './store.ts';
function fmt(ms: number): string {
const d = new Date(ms);
@@ -106,6 +110,64 @@ async function exportSS(since: number): Promise<void> {
await exportLeaderboards([...new Set(PLAYER_IDS.flatMap((id) => ssHashes(id, since)))].sort());
}
/** Overlap of each incremental window, so scores posted late or with a skewed clock aren't missed. */
const SYNC_OVERLAP_S = 24 * 3600;
/**
* Phase 6: the daily run. Main account only; the legacy accounts are frozen. Every step skips
* what's already archived, so a rerun (or a run after a missed day) just fills the gap.
*/
async function sync(opts: { ratings: boolean; ssReplays: boolean }): Promise<number> {
const started = new Date();
// Token upkeep first: persist the rotated refresh token before anything else, and fail loudly.
const tokens = await refresh();
console.log(`Token refreshed; refresh chain valid until about ${fmt(tokens.refresh_expires_at)}\n`);
await exportProfile(MAIN_PLAYER_ID);
const blSince = Math.max(0, newestScoreTime(MAIN_PLAYER_ID) - SYNC_OVERLAP_S);
const newBL = await exportScores(MAIN_PLAYER_ID, blSince);
// The whole window, not just the new scores, so a replay or stats file that failed yesterday is retried.
const blWindow = archivedScores(blSince).filter((s) => s.playerId === MAIN_PLAYER_ID);
await exportReplays(blWindow);
await exportScoreStats(blWindow);
// A new score can change a leaderboard (status, ratings), so re-snapshot its hash.
const newBLHashes = [...new Set(newBL.map((s) => s.leaderboard.song.hash.toLowerCase()))].sort();
await exportLeaderboards(newBLHashes, true);
await exportSSPlayer(MAIN_PLAYER_ID);
const ssSince = Math.max(0, newestSSScoreTime(MAIN_PLAYER_ID) - SYNC_OVERLAP_S);
const newSS = await exportSSScores(MAIN_PLAYER_ID, ssSince);
const newSSHashes = [...new Set(newSS.map((it) => it.leaderboard.songHash.toLowerCase()))].sort();
await exportLeaderboards(newSSHashes.filter((h) => !newBLHashes.includes(h)));
if (opts.ssReplays) await exportSSReplays(MAIN_PLAYER_ID, ssSince);
// Maps for everything played in either window; existing zips are skipped.
const windowHashes = [...new Set([...blWindow.map((s) => s.leaderboard.song.hash.toLowerCase()), ...ssHashes(MAIN_PLAYER_ID, ssSince)])].sort();
await exportMaps(windowHashes);
const noZip = windowHashes.filter((h) => !exists(archivePath('beatsaver', 'zips', `${h}.zip`)));
if (opts.ratings) {
// Upstream rebalances ratings; a new dated snapshot is written only when a hash changed.
await exportLeaderboards(playedHashes(0), true);
}
const summary = {
lastRun: started.toISOString(),
finished: new Date().toISOString(),
newBeatLeaderScores: newBL.map((s) => s.id),
newScoreSaberScores: newSS.map((it) => it.score.id),
ratingsRefreshed: opts.ratings,
mapsWithoutZip: noZip
};
updateState('sync', summary);
console.log('');
console.log(`Sync done: ${newBL.length} new BeatLeader scores, ${newSS.length} new ScoreSaber scores${opts.ratings ? ', ratings refreshed' : ''}.`);
if (noZip.length) console.log(`Maps without a zip (see beatsaver/missing.json; the game's CustomLevels are the manual fallback): ${noZip.join(' ')}`);
const failures = readState().failures;
if (failures && Object.keys(failures).length) console.warn(`Recorded failures (retried on the next run): ${JSON.stringify(failures)}`);
return 0;
}
function report(since: number, mapsSince: number): void {
const count = (...p: string[]) => listFiles(archivePath(...p)).length;
const day = (t: number) => (t ? new Date(t * 1000).toISOString().slice(0, 10) : 'all time');
@@ -218,12 +280,17 @@ async function main(): Promise<number> {
return 0;
}
}
if (cmd === 'sync') {
// Sundays in the machine's timezone (America/Los_Angeles on incineroar).
const ratings = args.includes('--ratings') || new Date().getDay() === 0;
return sync({ ratings, ssReplays: !args.includes('--no-ss-replays') });
}
if (cmd === 'report') {
report(since, mapsSince);
return 0;
}
console.error('Usage: node scripts/archive/cli.ts <auth login|refresh|status | export bl|ss|ss-replays|maps|all | report> [--since YYYY-MM-DD|all] [--maps-since YYYY-MM-DD|all]');
console.error('Usage: node scripts/archive/cli.ts <auth login|refresh|status | export bl|ss|ss-replays|maps|all | report | sync [--ratings] [--no-ss-replays]> [--since YYYY-MM-DD|all] [--maps-since YYYY-MM-DD|all]');
return 2;
}