diff --git a/flake.nix b/flake.nix index 0e43f05..003d41c 100644 --- a/flake.nix +++ b/flake.nix @@ -44,12 +44,35 @@ mainProgram = "plebsaber-stream"; }; }; + # The plebleader archive CLI (scripts/archive, docs/plan.md): plain + # TypeScript with only Node built-ins, run through Node's type stripping. + mkArchiveCli = pkgs: + pkgs.stdenvNoCC.mkDerivation { + pname = "plebleader-archive"; + version = "0.1.0"; + src = ./scripts/archive; + nativeBuildInputs = [ pkgs.makeWrapper ]; + installPhase = '' + runHook preInstall + mkdir -p $out/lib/plebleader-archive + cp *.ts $out/lib/plebleader-archive/ + makeWrapper ${pkgs.nodejs_22}/bin/node $out/bin/plebleader-archive \ + --add-flags "--experimental-strip-types --disable-warning=ExperimentalWarning" \ + --add-flags "$out/lib/plebleader-archive/cli.ts" + runHook postInstall + ''; + meta = { + description = "Archive exporter and daily sync for the personal BeatLeader mirror"; + mainProgram = "plebleader-archive"; + }; + }; in rec { packages = forAllSystems (system: let pkgs = import nixpkgs { inherit system; }; pkg = mkPackage pkgs; in { plebsaber-stream = pkg; + plebleader-archive = mkArchiveCli pkgs; default = pkg; }); diff --git a/scripts/archive/beatleader.ts b/scripts/archive/beatleader.ts index fd0bb1c..177ec67 100644 --- a/scripts/archive/beatleader.ts +++ b/scripts/archive/beatleader.ts @@ -3,11 +3,11 @@ import * as path from 'node:path'; import { BL_API } from './config.ts'; import { get, limiters } from './http.ts'; -import { archivePath, eachWithProgress, exists, listFiles, readJson, save, saveSnapshotIfChanged, today, updateState } from './store.ts'; +import { archivePath, eachWithProgress, exists, listFiles, readJson, readState, save, saveSnapshotIfChanged, today, updateState } from './store.ts'; const bl = (...p: string[]) => archivePath('beatleader', ...p); -type BLScore = { id: number; timepost: number; replay?: string | null; leaderboard: { song: { hash: string } } }; +export type BLScore = { id: number; playerId: string; timepost: number; replay?: string | null; leaderboard: { song: { hash: string } } }; async function blGet(pathAndQuery: string, accept?: number[]) { return get(`${BL_API}${pathAndQuery}`, { limiter: limiters.beatleader, accept }); @@ -39,12 +39,12 @@ export async function exportProfile(playerId: string): Promise { } } -/** Page through best scores newest-first from `sinceUnix`, saving each score object. */ -export async function exportScores(playerId: string, sinceUnix: number): Promise { +/** Page through best scores newest-first from `sinceUnix`, saving each score object. Returns the new ones. */ +export async function exportScores(playerId: string, sinceUnix: number): Promise { console.log(`BeatLeader scores for ${playerId} since ${new Date(sinceUnix * 1000).toISOString()}`); - let added = 0; + const added: BLScore[] = []; let seen = 0; - let newest = 0; + let newest = Number(readState()[`beatleader:${playerId}`]?.newestTimepost ?? 0); for (let page = 1; ; page++) { const qs = `count=100&page=${page}&sortBy=date&order=desc&includeIO=true&time_from=${sinceUnix}`; const res = await blGet(`/player/${playerId}/scores?${qs}`); @@ -56,12 +56,18 @@ export async function exportScores(playerId: string, sinceUnix: number): Promise if (exists(file)) continue; // The page is parsed and each element re-serialized: same content, no reshaping. save(file, JSON.stringify(score)); - added++; + added.push(score); } - console.log(` page ${page}: ${seen}/${json.metadata.total} seen, ${added} new`); + console.log(` page ${page}: ${seen}/${json.metadata.total} seen, ${added.length} new`); if (json.data.length < 100 || seen >= json.metadata.total) break; } updateState(`beatleader:${playerId}`, { scoresSince: sinceUnix, newestTimepost: newest, lastScoresRun: new Date().toISOString() }); + return added; +} + +/** Newest archived score time for a player (unix seconds), 0 when there is none. */ +export function newestScoreTime(playerId: string): number { + return archivedScores().reduce((max, s) => (s.playerId === playerId ? Math.max(max, s.timepost) : max), 0); } export async function exportReplays(scores: BLScore[]): Promise { diff --git a/scripts/archive/cli.ts b/scripts/archive/cli.ts index e03ab43..2542b9b 100644 --- a/scripts/archive/cli.ts +++ b/scripts/archive/cli.ts @@ -9,6 +9,10 @@ // plus all legacy ScoreSaber maps) // export all [--since D] bl, ss, maps (not ss-replays) // report [--since D] counts and gaps in the archive +// sync [--ratings] [--no-ss-replays] +// daily incremental run (phase 6): token refresh, new BL and SS +// scores with their replays, stats, leaderboards and maps. +// --ratings re-snapshots every played hash (default on Sundays) // // --since: YYYY-MM-DD, unix seconds, or "all" (default: all). --maps-since defaults to the two-year // window (DEFAULT_MAPS_SINCE). Every command skips files that already exist, so reruns only fill gaps. @@ -16,10 +20,10 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { ARCHIVE_DIR, BL_API, DEFAULT_MAPS_SINCE, LEGACY_PLAYER_ID, MAIN_PLAYER_ID, PLAYER_IDS, TOKENS_FILE, USER_AGENT, loadDotEnv, parseSince } from './config.ts'; import { getAccessToken, login, readTokens, refresh } from './auth.ts'; -import { archivedScores, blHashes, exportLeaderboards, exportProfile, exportReplays, exportScoreStats, exportScores } from './beatleader.ts'; -import { archivedSSScores, exportSSPlayer, exportSSReplays, exportSSScores, ssHashes } from './scoresaber.ts'; +import { archivedScores, blHashes, exportLeaderboards, exportProfile, exportReplays, exportScoreStats, exportScores, newestScoreTime } from './beatleader.ts'; +import { archivedSSScores, exportSSPlayer, exportSSReplays, exportSSScores, newestSSScoreTime, ssHashes } from './scoresaber.ts'; import { exportMaps } from './beatsaver.ts'; -import { archivePath, exists, listFiles, readState } from './store.ts'; +import { archivePath, exists, listFiles, readState, updateState } from './store.ts'; function fmt(ms: number): string { const d = new Date(ms); @@ -106,6 +110,64 @@ async function exportSS(since: number): Promise { await exportLeaderboards([...new Set(PLAYER_IDS.flatMap((id) => ssHashes(id, since)))].sort()); } +/** Overlap of each incremental window, so scores posted late or with a skewed clock aren't missed. */ +const SYNC_OVERLAP_S = 24 * 3600; + +/** + * Phase 6: the daily run. Main account only; the legacy accounts are frozen. Every step skips + * what's already archived, so a rerun (or a run after a missed day) just fills the gap. + */ +async function sync(opts: { ratings: boolean; ssReplays: boolean }): Promise { + const started = new Date(); + // Token upkeep first: persist the rotated refresh token before anything else, and fail loudly. + const tokens = await refresh(); + console.log(`Token refreshed; refresh chain valid until about ${fmt(tokens.refresh_expires_at)}\n`); + + await exportProfile(MAIN_PLAYER_ID); + const blSince = Math.max(0, newestScoreTime(MAIN_PLAYER_ID) - SYNC_OVERLAP_S); + const newBL = await exportScores(MAIN_PLAYER_ID, blSince); + // The whole window, not just the new scores, so a replay or stats file that failed yesterday is retried. + const blWindow = archivedScores(blSince).filter((s) => s.playerId === MAIN_PLAYER_ID); + await exportReplays(blWindow); + await exportScoreStats(blWindow); + // A new score can change a leaderboard (status, ratings), so re-snapshot its hash. + const newBLHashes = [...new Set(newBL.map((s) => s.leaderboard.song.hash.toLowerCase()))].sort(); + await exportLeaderboards(newBLHashes, true); + + await exportSSPlayer(MAIN_PLAYER_ID); + const ssSince = Math.max(0, newestSSScoreTime(MAIN_PLAYER_ID) - SYNC_OVERLAP_S); + const newSS = await exportSSScores(MAIN_PLAYER_ID, ssSince); + const newSSHashes = [...new Set(newSS.map((it) => it.leaderboard.songHash.toLowerCase()))].sort(); + await exportLeaderboards(newSSHashes.filter((h) => !newBLHashes.includes(h))); + if (opts.ssReplays) await exportSSReplays(MAIN_PLAYER_ID, ssSince); + + // Maps for everything played in either window; existing zips are skipped. + const windowHashes = [...new Set([...blWindow.map((s) => s.leaderboard.song.hash.toLowerCase()), ...ssHashes(MAIN_PLAYER_ID, ssSince)])].sort(); + await exportMaps(windowHashes); + const noZip = windowHashes.filter((h) => !exists(archivePath('beatsaver', 'zips', `${h}.zip`))); + + if (opts.ratings) { + // Upstream rebalances ratings; a new dated snapshot is written only when a hash changed. + await exportLeaderboards(playedHashes(0), true); + } + + const summary = { + lastRun: started.toISOString(), + finished: new Date().toISOString(), + newBeatLeaderScores: newBL.map((s) => s.id), + newScoreSaberScores: newSS.map((it) => it.score.id), + ratingsRefreshed: opts.ratings, + mapsWithoutZip: noZip + }; + updateState('sync', summary); + console.log(''); + console.log(`Sync done: ${newBL.length} new BeatLeader scores, ${newSS.length} new ScoreSaber scores${opts.ratings ? ', ratings refreshed' : ''}.`); + if (noZip.length) console.log(`Maps without a zip (see beatsaver/missing.json; the game's CustomLevels are the manual fallback): ${noZip.join(' ')}`); + const failures = readState().failures; + if (failures && Object.keys(failures).length) console.warn(`Recorded failures (retried on the next run): ${JSON.stringify(failures)}`); + return 0; +} + function report(since: number, mapsSince: number): void { const count = (...p: string[]) => listFiles(archivePath(...p)).length; const day = (t: number) => (t ? new Date(t * 1000).toISOString().slice(0, 10) : 'all time'); @@ -218,12 +280,17 @@ async function main(): Promise { return 0; } } + if (cmd === 'sync') { + // Sundays in the machine's timezone (America/Los_Angeles on incineroar). + const ratings = args.includes('--ratings') || new Date().getDay() === 0; + return sync({ ratings, ssReplays: !args.includes('--no-ss-replays') }); + } if (cmd === 'report') { report(since, mapsSince); return 0; } - console.error('Usage: node scripts/archive/cli.ts [--since YYYY-MM-DD|all] [--maps-since YYYY-MM-DD|all]'); + console.error('Usage: node scripts/archive/cli.ts [--since YYYY-MM-DD|all] [--maps-since YYYY-MM-DD|all]'); return 2; } diff --git a/scripts/archive/scoresaber.ts b/scripts/archive/scoresaber.ts index 7a2cf96..a03613d 100644 --- a/scripts/archive/scoresaber.ts +++ b/scripts/archive/scoresaber.ts @@ -26,10 +26,10 @@ export async function exportSSPlayer(playerId: string): Promise { console.log(`ScoreSaber player ${playerId}: ${saveSnapshotIfChanged(ss(playerId, 'player'), r.body) ? 'saved' : 'unchanged'}`); } -/** Page `sort=recent` until scores are older than `sinceUnix`. */ -export async function exportSSScores(playerId: string, sinceUnix: number): Promise { +/** Page `sort=recent` until scores are older than `sinceUnix`. Returns the new items. */ +export async function exportSSScores(playerId: string, sinceUnix: number): Promise { console.log(`ScoreSaber scores for ${playerId} since ${new Date(sinceUnix * 1000).toISOString()}`); - let added = 0; + const added: SSItem[] = []; let seen = 0; for (let page = 1; ; page++) { const r = await get(`${SS_API}/player/${playerId}/scores?limit=100&page=${page}&sort=recent`, { limiter: limiters.scoresaber }); @@ -44,12 +44,18 @@ export async function exportSSScores(playerId: string, sinceUnix: number): Promi const file = ss(playerId, 'scores', `${item.score.id}.json`); if (exists(file)) continue; save(file, JSON.stringify(item)); - added++; + added.push(item); } - console.log(` page ${page}: ${seen} in window, ${added} new (total on server ${json.metadata.total})`); + console.log(` page ${page}: ${seen} in window, ${added.length} new (total on server ${json.metadata.total})`); if (reachedCutoff || json.playerScores.length < 100) break; } updateState(`scoresaber:${playerId}`, { scoresSince: sinceUnix, lastScoresRun: new Date().toISOString() }); + return added; +} + +/** Newest archived ScoreSaber score time for a player (unix seconds), 0 when there is none. */ +export function newestSSScoreTime(playerId: string): number { + return archivedSSScores(playerId).reduce((max, it) => Math.max(max, Math.floor(Date.parse(it.score.timeSet) / 1000)), 0); } /** Download replays for archived scores with hasReplay, oldest first, ~10.5 s apart. */