From a759308711cd341158123624a59050869d99fe52 Mon Sep 17 00:00:00 2001 From: pleb Date: Sun, 8 Nov 2026 18:30:02 -0800 Subject: [PATCH] Add archive exporter for the personal BeatLeader mirror scripts/archive is a CLI that exports BeatLeader and ScoreSaber scores, replays, score stats, leaderboard snapshots and BeatSaver map zips into an append-only archive. Anchor the .gitignore 'archive' rule to the repo root so it no longer hides scripts/archive. Record phase 1 results in the plan. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 2 +- docs/plan.md | 126 +++++++++++++----- scripts/archive/auth.ts | 192 +++++++++++++++++++++++++++ scripts/archive/beatleader.ts | 116 +++++++++++++++++ scripts/archive/beatsaver.ts | 89 +++++++++++++ scripts/archive/cli.ts | 236 ++++++++++++++++++++++++++++++++++ scripts/archive/config.ts | 76 +++++++++++ scripts/archive/http.ts | 107 +++++++++++++++ scripts/archive/scoresaber.ts | 101 +++++++++++++++ scripts/archive/store.ts | 80 ++++++++++++ 10 files changed, 1093 insertions(+), 32 deletions(-) create mode 100644 scripts/archive/auth.ts create mode 100644 scripts/archive/beatleader.ts create mode 100644 scripts/archive/beatsaver.ts create mode 100644 scripts/archive/cli.ts create mode 100644 scripts/archive/config.ts create mode 100644 scripts/archive/http.ts create mode 100644 scripts/archive/scoresaber.ts create mode 100644 scripts/archive/store.ts diff --git a/.gitignore b/.gitignore index 4464c3f..42515f0 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,4 @@ vite.config.ts.timestamp-* .data .env -archive \ No newline at end of file +/archive diff --git a/docs/plan.md b/docs/plan.md index ae64148..8e11eef 100644 --- a/docs/plan.md +++ b/docs/plan.md @@ -17,7 +17,7 @@ Written 2026-11-08. The numbers under "Facts" were checked live on that date. | Replay viewer | Self-host ArcViewer's prebuilt WebGL build (the `deploy` branch) for both BeatLeader (`.bsor`) and ScoreSaber (`.dat`) replays. No Unity build. | | Source of truth | A raw, append-only **archive** of unmodified API responses and replay files. The SQLite database can always be rebuilt from it, and so can any future replacement system. | | Exporter and sync | TypeScript in this repo, building on `src/lib/server/beatleader.ts`, `beatleaderAuth.ts`, `scoresaber.ts` and `beatsaver.ts`. It writes only to the archive. | -| BeatLeader auth | OAuth 2.0 (`profile offline_access`), the same flow as plebsaber.stream's login, which already unlocks unranked ratings. The sync owns its own chain of refresh tokens. | +| BeatLeader auth | OAuth 2.0 (`profile offline_access`). The sync owns its own chain of refresh tokens (`scripts/archive/cli.ts auth login`, seeded 2026-11-08). Unranked ratings do **not** need it: they come from the anonymous `/leaderboards/hash/{hash}` (see Facts). | | Map files | Archive the BeatSaver zip of every map I've played, using `beatsaver.ts`. Unpack them into the server's `MapsPath`, so the server and ArcViewer never need BeatSaver. | | Sync schedule | Daily at 17:00 America/Los_Angeles, incineroar's timezone. | | Access | Only incineroar and snorlax, through Caddy's `remote_ip` allowlist. | @@ -49,23 +49,32 @@ player/hash/difficulty/characteristic, so it can recover some scores whose server copy has `hasReplay: false`. `~/ops/beatsaber/backups` deliberately leaves these folders out. -### Unranked ratings need an authenticated request +### Unranked ratings and auth (corrected 2026-11-08) - Every difficulty object has `stars`, `passRating`, `accRating`, `techRating` (the skill triangle), `predictedAcc`, plus `modifiersRating` with `ss*`/`fs*`/`sf*`/`bfs*`/`bsf*` variants of each rating. -- For **unranked** maps the server fills these in only when the *requesting* - user has `profileSettings.showAllRatings == true`. Mine does. An anonymous - request for the same score returns `stars/pass/acc/techRating: null`; only - `predictedAcc` comes through. See - `Controllers/PlayerScoresController.cs` (`showRatings = currentProfileSettings?.ShowAllRatings`). -- **The solution is OAuth 2.0, as in plebsaber.stream.** Logging in through - `/auth/beatleader/login` requests `scp:profile scp:offline_access`. Requests - sent with `Authorization: Bearer ` (`createBeatLeaderAPI(fetch, - token)`) come back with unranked ratings filled in. The code is in - `src/lib/server/beatleaderAuth.ts`: `exchangeCodeForTokens`, - `refreshAccessToken`, `getValidAccessToken`, and the token store - `beatleader_tokens.json` in the data dir. +- On **score lists** (`/player/{id}/scores`) and `/leaderboard/{id}`, unranked + ratings are filled in only when the *requesting* user has + `profileSettings.showAllRatings == true`. That check uses + `HttpContext.CurrentUserID`, which reads only the beatleader.com **cookie + session** (with an `Issued` claim). An OAuth bearer token does not count: + tested, bearer and anonymous both return `stars/pass/acc/techRating: null`. +- **`GET /leaderboards/hash/{hash}` returns full ratings for unranked + difficulties, anonymously**, including all `modifiersRating` variants. + Tested on an unranked ExpertPlus (`9db01c9b…`): stars 8.33, pass 3.92, acc + 9.75, tech 7.70. Phase 1, step 6 already fetches this for every played hash, + so the importer takes difficulty ratings from these snapshots, not from the + nulls embedded in the scores. +- Bearer tokens only work on `/oauth2/identity`, clan management and + ReeSabers presets (`CurrentOauthUserID`). `GET /user` with a bearer token is + 401. The exporter doesn't actually need auth; the token chain is kept for + identity checks and in case a cookie-gated endpoint turns out to matter. +- The sync's chain lives in `scripts/archive/auth.ts`: login via the loopback + redirect `http://localhost:5173/auth/beatleader/callback` (registered on the + OAuth app, as is `https://plebsaber.stream/auth/beatleader/callback`), token + store `/auth/beatleader_tokens.json` (default + `.data/plebleader/`, mode 0600). `auth status` checks identity and ratings. - Token lifetimes: access tokens last 3,600 s and refresh tokens 14 days. Each refresh **rotates** the refresh token and invalidates the old one. So the sync job needs its **own** refresh token, separate from the @@ -111,8 +120,12 @@ ScoreSaber (public, no auth): `GET https://scoresaber.com/api/player/{id}/full`, `score.hasReplay`), and `GET /api/v2/scores/{scoreId}/replay` (`.dat`). The ScoreSaber API doesn't send CORS headers, so it only works server-side. `~/ops/beatsaber/ss-scraper/download_replays.py` already handles pacing -(30 s minimum between replays), 429 backoff, and resuming. At that pace the -1,034 main-account replays take about 9 hours. +(30 s minimum between replays), 429 backoff, and resuming. That pacing is +outdated: as of 2026-11-08 ScoreSaber sends `x-ratelimit-{limit,remaining,reset}-{short,medium,long}` +headers. The replay endpoint allows 20 (short), 60/min (medium) and +**360/hour** (long, the one that binds); player endpoints allow 25/100/400. +`scripts/archive` paces replays at 10.5 s and pauses whenever a window's +remaining count hits 0, so 1,034 replays take about 3 h instead of 9. ### Upstream server: what the fork has to change @@ -192,7 +205,8 @@ avoids CORS and mixed-content problems. - Disk: `/` has 479 GB free and `/home` 684 GB. The archive should be 10–25 GB: about 4.6 GB of best-score replays, about 1.4 GB of ScoreSaber replays, and BeatSaver zips for roughly 2,000–3,000 played maps (ScoreSaber alone has 1,474 unique hashes), - at a few MB each. + at a few MB each. Actual (2026-11-08): 21 GB with 1,897 zips at about 8 MB + each. Fetching the remaining ~630 older maps later adds about 5 GB. - Allowed clients: incineroar (`127.0.0.1/32`, `::1/128`, `192.168.1.36/32`) and snorlax (`192.168.254.21/32`, the address the amelia fallback vhost already allows). @@ -265,7 +279,9 @@ the fork, so do it first. plebsaber.stream; check that its registered callback URLs include the one used for seeding. Agents never decrypt it. For local runs before deployment, use the gitignored `.env` in this repo. -5. **Seed the sync's token chain.** Add a small `scripts/archive` command +5. **Done 2026-11-08** (`scripts/archive/cli.ts auth login`; refresh rotation + tested; chain expires about 2026-11-22 unless refreshed). + **Seed the sync's token chain.** Add a small `scripts/archive` command (`auth login`) that runs the same authorization-code flow as `/auth/beatleader/login` with `scope=profile offline_access`. It catches the callback on a loopback port, or reuses a plebsaber.stream callback URL the @@ -275,6 +291,26 @@ the fork, so do it first. ### Phase 1: Full export (one-time backup) +**Scope (decided 2026-11-08):** all player data for both accounts, all +history: profiles, BL scores, replays, score stats, SS scores and replays, +plus BL leaderboard JSON (ratings) for every played hash. Only **BeatSaver +map files** are limited to maps played in the last two years +(`--maps-since 2024-11-08`, `DEFAULT_MAPS_SINCE` in `scripts/archive/config.ts`). +**Exception:** legacy ScoreSaber scores that have an archived replay get their +map fetched whatever its age, so those replays render (58 replays, 53 maps; +`mapHashes()` in `cli.ts`). Legacy scores without a replay don't get a map. Older map zips come later with `export maps --maps-since all`; existing files +are skipped, so that only fetches what's missing. Replays of older scores +won't render a map in ArcViewer until then. SS main has 1,034 replays (about +3 h at 10.5 s each). The 58 SS legacy replays are copied from ss-scraper's +download instead of being fetched again. + +Commands: `node scripts/archive/cli.ts export bl|ss|ss-replays|maps|all +[--since …]` and `report`. The archive lives in `.data/plebleader/archive/` +(set `PLEBLEADER_DIR` to change the root) until `/var/lib/plebleader` exists +on incineroar; then move it. Logs go in `.data/plebleader/logs/`. +Profile, history, triangle and accgraph are stored as dated snapshots in +`players///.json`, written only when changed. + Build `scripts/archive/` as a small CLI (`export bl`, `export ss`, `export maps`, `import-local-mods`, `sync`). Rules: @@ -283,14 +319,15 @@ Build `scripts/archive/` as a small CLI (`export bl`, `export ss`, `state.json`, and write to temp files before renaming. - Respect rate limits. Use about 4 requests per second to BeatLeader and honor `Retry-After` (the existing `requestWith429Retry` handles 429s). For - ScoreSaber replays, keep ss-scraper's 30 s pacing. + ScoreSaber replays, stay under the 360/hour limit (10.5 s pacing) and honor + the `x-ratelimit-*` headers. - Send `User-Agent: plebsaber-archive/0.1`. Don't put an email address in it. Steps: -1. **Auth check.** With the bearer token, `GET /user` must return `player.id == - 76561199407393962`, and an unranked score's `difficulty.stars` must not be - null. If either fails, stop. +1. **Auth check.** `node scripts/archive/cli.ts auth status`: `/oauth2/identity` + must return `76561199407393962`, and `/leaderboards/hash/{hash}` must return + non-null `stars` for an unranked difficulty. If either fails, stop. 2. **BeatLeader scores.** Page through `/player/{id}/scores` with `sortBy=date&order=desc&count=100&includeIO=true` until it's exhausted (about 36 pages). Do the same for the legacy id (3 scores). @@ -301,8 +338,9 @@ Steps: 6. **BeatLeader map data.** Collect the set of played hashes: every unique `song.hash` in BL scores, plus `leaderboard.songHash` in both ScoreSaber accounts. Normalize them to lowercase. For each, call - `GET /leaderboards/hash/{hash}` with the bearer token. That gives ratings - and the triangle for every difficulty, ranked or not. + `GET /leaderboards/hash/{hash}` (no auth needed). That gives ratings and + the triangle for every difficulty, ranked or not. This is the **only** + source of unranked ratings; the scores from step 2 have them nulled. 7. **Map files from BeatSaver.** For each played hash, call `BeatSaverAPI.getMapByHash(hash)` and save the response to `beatsaver/maps/.json`. Then download the zip for the `versions[]` @@ -311,11 +349,16 @@ Steps: checks it's a valid zip containing `Info.dat`, then renames it. Go about 2 requests per second and honor 429s. Write each hash that can't be resolved (deleted map, or an old version gone from the CDN) to `missing.json`. -8. **Recover missing maps from the game.** With the Windows partition mounted - (`mountwin`), hash every folder in the BSManager instance's - `Beat Saber_Data/CustomLevels` and `CustomWIPLevels`. Copy the folders whose - hash is in `missing.json` to `game-levels//`. Whatever is still - missing stays listed; those replays can't render the map. +8. **Recover missing maps from the game: optional, low priority.** Decided + 2026-11-08: maps gone from BeatSaver are accepted as missing. They were most + likely updated or removed for a reason. The user's online backups and local + map archives from before 2026 don't have them. Older cold storage might, so + that's a "someday" check, not a task. The current BSManager instance's + `CustomLevels` may hold a few recent ones and is cheap to try: with + `mountwin`, hash every folder in `Beat Saber_Data/CustomLevels` and + `CustomWIPLevels`, and copy those whose hash is in `missing.json` to + `game-levels//`. Whatever is still missing stays listed; those replays + just can't render the map. 9. **Unpack for the server.** Extract each zip, or copy each recovered folder, to `maps//`. Then the server's `MapDownloader` never goes to the network, and ArcViewer can load `url=/cdn/maps/.zip` (serve the zips) @@ -343,6 +386,26 @@ leaderboard snapshot, plus a BeatSaver zip or recovered game folder, unless it's listed in `missing.json`. Write a `report` subcommand that prints counts and what's missing, and paste its output into this file. +**Report, 2026-11-08** (ScoreSaber main replays still downloading): + +``` +Window: scores since all time, BeatSaver maps since 2024-11-08 +BeatLeader scores 3568 in window (3568 archived total, both accounts) +BeatLeader replays 3568/3568 +BeatLeader score stats 3568/3568 +ScoreSaber 76561199407393962: 2107 scores, replays 435/1034 +ScoreSaber 76561197962107242: 199 scores, replays 58/58 +Played hashes 2560 + BL leaderboard snapshot 2552/2560 + BeatSaver zip 1897/1926 in maps window (missing.json lists 29) +Archive size 21.15 GB +``` + +The 29 missing hashes are all "hash unknown to BeatSaver", and none belongs to +a legacy score with a replay (all 53 of those maps are archived). The 8 hashes +without a BL leaderboard are ScoreSaber-only maps BeatLeader never saw. Zips +average about 8 MB (15 GB for 1,897), more than the estimate below. + ### Phase 2: Server fork on SQLite Work on the `pleb/main` branch of `pleb/beatleader-server`. Keep each change @@ -607,8 +670,9 @@ the local zip. far. Sync stops, a failing unit makes that visible, and `auth login` re-seeds the chain. - **Maps disappear from BeatSaver.** That's why zips are archived as soon as a - hash first appears. Old versions that are already gone come from the game's - own folders, or stay listed in `missing.json`. + hash first appears. Old versions that are already gone stay listed in + `missing.json`, which is acceptable (decided 2026-11-08). Cold storage might + hold some; check someday, without much hope. - **Upstream API changes.** The archive stores raw responses, so a change breaks the importer, not the data. Version the importer's mapping. - **Upstream code moves fast** (the website is updated daily). Pin fork diff --git a/scripts/archive/auth.ts b/scripts/archive/auth.ts new file mode 100644 index 0000000..9577a7f --- /dev/null +++ b/scripts/archive/auth.ts @@ -0,0 +1,192 @@ +// BeatLeader OAuth 2.0 token chain owned by the archive sync. +// +// This chain is separate from plebsaber.stream's own login: BeatLeader rotates +// the refresh token on every refresh and invalidates the old one, so two +// consumers sharing one chain would break each other. +import * as fs from 'node:fs'; +import * as http from 'node:http'; +import { randomUUID } from 'node:crypto'; +import { BL_API, TOKENS_FILE, USER_AGENT, clientCredentials, writeFileAtomic } from './config.ts'; + +const AUTHORIZE_URL = `${BL_API}/oauth2/authorize`; +const TOKEN_URL = `${BL_API}/oauth2/token`; +const SCOPES = 'profile offline_access'; +// Registered redirect URIs on the OAuth app (verified 2026-11-08): +// https://plebsaber.stream/auth/beatleader/callback +// http://localhost:5173/auth/beatleader/callback +const LOOPBACK_PORT = 5173; +const LOOPBACK_REDIRECT = `http://localhost:${LOOPBACK_PORT}/auth/beatleader/callback`; +const REFRESH_TOKEN_LIFETIME_MS = 14 * 24 * 3600 * 1000; + +export type TokenStore = { + access_token: string; + refresh_token: string; + /** epoch ms */ + access_expires_at: number; + /** epoch ms, estimated from BeatLeader's 14-day refresh token lifetime */ + refresh_expires_at: number; + /** epoch ms of the last token grant/refresh */ + updated_at: number; + player_id?: string; + player_name?: string; +}; + +type TokenResponse = { access_token: string; refresh_token?: string; expires_in?: number; token_type?: string }; + +export function readTokens(): TokenStore | null { + try { + return JSON.parse(fs.readFileSync(TOKENS_FILE, 'utf-8')) as TokenStore; + } catch { + return null; + } +} + +function saveTokens(res: TokenResponse, previous: TokenStore | null, identity?: { id: string; name?: string }): TokenStore { + if (!res.refresh_token) { + throw new Error('Token response has no refresh_token; is offline_access granted?'); + } + const now = Date.now(); + const store: TokenStore = { + access_token: res.access_token, + refresh_token: res.refresh_token, + access_expires_at: now + (res.expires_in ?? 3600) * 1000, + refresh_expires_at: now + REFRESH_TOKEN_LIFETIME_MS, + updated_at: now, + player_id: identity?.id ?? previous?.player_id, + player_name: identity?.name ?? previous?.player_name + }; + // Persist the rotated refresh token before anything uses the new access token. + writeFileAtomic(TOKENS_FILE, JSON.stringify(store, null, 2) + '\n', 0o600); + return store; +} + +async function tokenRequest(params: Record): Promise { + const { clientId, clientSecret } = clientCredentials(); + const res = await fetch(TOKEN_URL, { + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded', 'User-Agent': USER_AGENT }, + body: new URLSearchParams({ client_id: clientId, client_secret: clientSecret, ...params }) + }); + const text = await res.text(); + if (!res.ok) throw new Error(`Token request failed: HTTP ${res.status} ${text.slice(0, 300)}`); + return JSON.parse(text) as TokenResponse; +} + +export async function fetchIdentity(accessToken: string): Promise<{ id: string; name?: string }> { + const res = await fetch(`${BL_API}/oauth2/identity`, { + headers: { Authorization: `Bearer ${accessToken}`, 'User-Agent': USER_AGENT } + }); + if (!res.ok) throw new Error(`Identity request failed: HTTP ${res.status}`); + const json = (await res.json()) as { id?: string; name?: string }; + if (!json.id) throw new Error('Identity response has no id'); + return { id: json.id, name: json.name }; +} + +/** Wait for the OAuth redirect on the loopback port, or for a pasted redirect URL on stdin. */ +function waitForCode(state: string): Promise { + return new Promise((resolve, reject) => { + let done = false; + const finish = (err: Error | null, code?: string) => { + if (done) return; + done = true; + server.close(); + process.stdin.pause(); + if (err) reject(err); + else resolve(code!); + }; + + const fromUrl = (raw: string): string => { + const url = new URL(raw, LOOPBACK_REDIRECT); + const error = url.searchParams.get('error'); + if (error) throw new Error(`Authorization denied: ${error} ${url.searchParams.get('error_description') ?? ''}`); + const code = url.searchParams.get('code'); + if (!code) throw new Error('Redirect has no code parameter'); + if (url.searchParams.get('state') !== state) throw new Error('State mismatch; start the login again'); + return code; + }; + + const server = http.createServer((req, res) => { + if (!req.url?.startsWith('/auth/beatleader/callback')) { + res.writeHead(404).end(); + return; + } + try { + const code = fromUrl(req.url); + res.writeHead(200, { 'Content-Type': 'text/plain; charset=utf-8' }); + res.end('plebleader archive: BeatLeader authorization received. You can close this tab.\n'); + finish(null, code); + } catch (err) { + res.writeHead(400, { 'Content-Type': 'text/plain; charset=utf-8' }); + res.end(`plebleader archive: ${(err as Error).message}\n`); + finish(err as Error); + } + }); + server.on('error', (err: NodeJS.ErrnoException) => { + if (err.code === 'EADDRINUSE') { + console.error(`Port ${LOOPBACK_PORT} is busy (is \`vite dev\` running?). Paste the redirected URL below instead.`); + } else { + finish(err); + } + }); + server.listen(LOOPBACK_PORT, 'localhost'); + + // Fallback: paste the full redirected URL (e.g. when the browser is on another machine). + process.stdin.setEncoding('utf-8'); + process.stdin.on('data', (chunk: string) => { + const line = chunk.trim(); + if (!line) return; + try { + finish(null, fromUrl(line)); + } catch (err) { + console.error((err as Error).message); + } + }); + process.stdin.resume(); + }); +} + +export async function login(): Promise { + const { clientId } = clientCredentials(); + const state = randomUUID(); + const url = new URL(AUTHORIZE_URL); + url.searchParams.set('client_id', clientId); + url.searchParams.set('scope', SCOPES); + url.searchParams.set('response_type', 'code'); + url.searchParams.set('redirect_uri', LOOPBACK_REDIRECT); + url.searchParams.set('prompt', 'consent'); + url.searchParams.set('state', state); + + console.log('Open this URL in a browser on this machine and approve access:\n'); + console.log(` ${url.toString()}\n`); + console.log(`Listening on ${LOOPBACK_REDIRECT}`); + console.log('If the browser is on another machine, paste the URL it was redirected to (the page will fail to load) and press Enter.\n'); + + const code = await waitForCode(state); + const res = await tokenRequest({ grant_type: 'authorization_code', code, redirect_uri: LOOPBACK_REDIRECT }); + // Save immediately so the refresh token survives even if the identity lookup fails. + let store = saveTokens(res, readTokens()); + const identity = await fetchIdentity(store.access_token); + store = { ...store, player_id: identity.id, player_name: identity.name }; + writeFileAtomic(TOKENS_FILE, JSON.stringify(store, null, 2) + '\n', 0o600); + return store; +} + +export async function refresh(): Promise { + const current = readTokens(); + if (!current?.refresh_token) { + throw new Error(`No refresh token in ${TOKENS_FILE}. Run \`auth login\` first.`); + } + try { + const res = await tokenRequest({ grant_type: 'refresh_token', refresh_token: current.refresh_token }); + return saveTokens(res, current); + } catch (err) { + throw new Error(`${(err as Error).message}\nThe refresh chain is broken. Run \`node scripts/archive/cli.ts auth login\` to seed it again.`); + } +} + +/** A valid access token, refreshing (and rotating the stored refresh token) when needed. */ +export async function getAccessToken(minValidityMs = 5 * 60 * 1000): Promise { + const current = readTokens(); + if (current && current.access_expires_at - Date.now() > minValidityMs) return current.access_token; + return (await refresh()).access_token; +} diff --git a/scripts/archive/beatleader.ts b/scripts/archive/beatleader.ts new file mode 100644 index 0000000..01f7742 --- /dev/null +++ b/scripts/archive/beatleader.ts @@ -0,0 +1,116 @@ +// BeatLeader exporter: scores, replays, score stats, leaderboards per played hash, profile. +// Writes only to archive/beatleader/. Everything is resumable: existing files are skipped. +import * as path from 'node:path'; +import { BL_API } from './config.ts'; +import { get, limiters } from './http.ts'; +import { archivePath, eachWithProgress, exists, listFiles, readJson, save, saveSnapshotIfChanged, today, updateState } from './store.ts'; + +const bl = (...p: string[]) => archivePath('beatleader', ...p); + +type BLScore = { id: number; timepost: number; replay?: string | null; leaderboard: { song: { hash: string } } }; + +async function blGet(pathAndQuery: string, accept?: number[]) { + return get(`${BL_API}${pathAndQuery}`, { limiter: limiters.beatleader, accept }); +} + +/** Archived BL scores (parsed), optionally only those at or after `sinceUnix`. */ +export function archivedScores(sinceUnix = 0): BLScore[] { + const dir = bl('scores'); + return listFiles(dir, '.json') + .map((f) => readJson(path.join(dir, f))) + .filter((s) => s.timepost >= sinceUnix); +} + +export async function exportProfile(playerId: string): Promise { + console.log(`BeatLeader profile ${playerId}`); + const dir = (kind: string) => bl('players', playerId, kind); + const endpoints: [string, string][] = [ + ['profile', `/player/${playerId}?stats=true`], + ['history', `/player/${playerId}/history`], + ['triangle', `/player/${playerId}/history/triangle`], + ['accgraph', `/player/${playerId}/accgraph`] + ]; + for (const [kind, url] of endpoints) { + const res = await blGet(url); + const wrote = saveSnapshotIfChanged(dir(kind), res.body); + console.log(` ${kind}: ${wrote ? `saved ${today()}` : 'unchanged'}`); + } +} + +/** Page through best scores newest-first from `sinceUnix`, saving each score object. */ +export async function exportScores(playerId: string, sinceUnix: number): Promise { + console.log(`BeatLeader scores for ${playerId} since ${new Date(sinceUnix * 1000).toISOString()}`); + let added = 0; + let seen = 0; + let newest = 0; + for (let page = 1; ; page++) { + const qs = `count=100&page=${page}&sortBy=date&order=desc&includeIO=true&time_from=${sinceUnix}`; + const res = await blGet(`/player/${playerId}/scores?${qs}`); + const json = JSON.parse(res.body.toString('utf-8')) as { metadata: { total: number }; data: BLScore[] }; + for (const score of json.data) { + seen++; + newest = Math.max(newest, score.timepost); + const file = bl('scores', `${score.id}.json`); + if (exists(file)) continue; + // The page is parsed and each element re-serialized: same content, no reshaping. + save(file, JSON.stringify(score)); + added++; + } + console.log(` page ${page}: ${seen}/${json.metadata.total} seen, ${added} new`); + if (json.data.length < 100 || seen >= json.metadata.total) break; + } + updateState(`beatleader:${playerId}`, { scoresSince: sinceUnix, newestTimepost: newest, lastScoresRun: new Date().toISOString() }); +} + +export async function exportReplays(scores: BLScore[]): Promise { + const withReplay = scores.filter((s) => s.replay); + console.log(`BeatLeader replays: ${withReplay.length} scores with a replay URL`); + const res = await eachWithProgress('replays', withReplay, async (s) => { + const name = decodeURIComponent(new URL(s.replay!).pathname.split('/').pop()!); + const file = bl('replays', name); + if (exists(file)) return 'skipped'; + const r = await get(s.replay!, { limiter: limiters.blCdn, accept: [404] }); + if (r.status === 404) return 'missing'; + save(file, r.body); + return 'done'; + }); + recordFailures('replays', res.failed.map((f) => f.item.id)); +} + +export async function exportScoreStats(scores: BLScore[]): Promise { + console.log(`BeatLeader score stats: ${scores.length} scores`); + const res = await eachWithProgress('scorestats', scores, async (s) => { + const file = bl('scorestats', `${s.id}.json`); + if (exists(file)) return 'skipped'; + // 302 to a presigned R2 URL; fetch follows it. + const r = await blGet(`/score/statistic/${s.id}`, [404]); + if (r.status === 404) return 'missing'; + save(file, r.body); + return 'done'; + }); + recordFailures('scorestats', res.failed.map((f) => f.item.id)); +} + +/** /leaderboards/hash/{hash}: all difficulties with full ratings (also unranked, no auth needed). */ +export async function exportLeaderboards(hashes: string[], refresh = false): Promise { + console.log(`BeatLeader leaderboards: ${hashes.length} hashes${refresh ? ' (refresh)' : ''}`); + const res = await eachWithProgress('leaderboards', hashes, async (hash) => { + const dir = bl('leaderboards', hash); + if (!refresh && listFiles(dir, '.json').length > 0) return 'skipped'; + const r = await blGet(`/leaderboards/hash/${hash}`, [404]); + if (r.status === 404) return 'missing'; + return saveSnapshotIfChanged(dir, r.body) ? 'done' : 'skipped'; + }); + recordFailures('leaderboards', res.failed.map((f) => f.item)); +} + +function recordFailures(kind: string, ids: (string | number)[]): void { + if (ids.length) { + updateState('failures', { [`beatleader:${kind}`]: ids }); + console.warn(` ${ids.length} ${kind} failed; rerun to retry`); + } +} + +export function blHashes(sinceUnix = 0): string[] { + return [...new Set(archivedScores(sinceUnix).map((s) => s.leaderboard.song.hash.toLowerCase()))].sort(); +} diff --git a/scripts/archive/beatsaver.ts b/scripts/archive/beatsaver.ts new file mode 100644 index 0000000..d722f33 --- /dev/null +++ b/scripts/archive/beatsaver.ts @@ -0,0 +1,89 @@ +// BeatSaver exporter: map JSON and the zip of the exact version hash that was played. +import * as fs from 'node:fs'; +import { get, limiters } from './http.ts'; +import { archivePath, eachWithProgress, exists, readJson, save } from './store.ts'; + +const bs = (...p: string[]) => archivePath('beatsaver', ...p); +const MISSING_FILE = () => bs('missing.json'); + +type BSMap = { id: string; versions: { hash: string; downloadURL: string }[] }; +type Missing = Record; + +function readMissing(): Missing { + try { + return readJson(MISSING_FILE()); + } catch { + return {}; + } +} + +/** Entry names from a zip's central directory, or null if the zip is truncated or malformed. */ +export function zipEntryNames(buf: Buffer): string[] | null { + if (buf.length < 22 || buf.readUInt32LE(0) !== 0x04034b50) return null; + const tailStart = Math.max(0, buf.length - 65_557); + const eocd = buf.subarray(tailStart).lastIndexOf(Buffer.from([0x50, 0x4b, 0x05, 0x06])); + if (eocd < 0) return null; + const e = tailStart + eocd; + const count = buf.readUInt16LE(e + 10); + let p = buf.readUInt32LE(e + 16); + const names: string[] = []; + for (let i = 0; i < count; i++) { + if (p + 46 > buf.length || buf.readUInt32LE(p) !== 0x02014b50) return null; + const nameLen = buf.readUInt16LE(p + 28); + const extraLen = buf.readUInt16LE(p + 30); + const commentLen = buf.readUInt16LE(p + 32); + names.push(buf.toString('utf-8', p + 46, p + 46 + nameLen)); + p += 46 + nameLen + extraLen + commentLen; + } + return names; +} + +/** A zip is usable if its central directory is intact and lists an Info.dat (any case, any folder). */ +export function looksLikeMapZip(buf: Buffer): boolean { + return zipEntryNames(buf)?.some((n) => /(^|\/)info\.dat$/i.test(n)) ?? false; +} + +export async function exportMaps(hashes: string[]): Promise { + const missing = readMissing(); + console.log(`BeatSaver maps: ${hashes.length} hashes`); + const res = await eachWithProgress('maps', hashes, async (hash) => { + const jsonFile = bs('maps', `${hash}.json`); + const zipFile = bs('zips', `${hash}.zip`); + if (exists(zipFile) && exists(jsonFile)) return 'skipped'; + + let map: BSMap | null = null; + if (exists(jsonFile)) { + map = readJson(jsonFile); + } else { + const r = await get(`https://api.beatsaver.com/maps/hash/${hash}`, { limiter: limiters.beatsaver, accept: [404] }); + if (r.status === 200) { + save(jsonFile, r.body); + map = JSON.parse(r.body.toString('utf-8')) as BSMap; + } + } + if (exists(zipFile)) return 'done'; + + // Prefer the version entry for this exact hash; old versions often remain on the CDN. + const version = map?.versions.find((v) => v.hash.toLowerCase() === hash); + const candidates = [version?.downloadURL, `https://r2cdn.beatsaver.com/${hash}.zip`, `https://cdn.beatsaver.com/${hash}.zip`].filter( + (u): u is string => Boolean(u) + ); + for (const url of [...new Set(candidates)]) { + const r = await get(url, { limiter: limiters.beatsaverCdn, accept: [403, 404] }); + if (r.status === 200 && looksLikeMapZip(r.body)) { + save(zipFile, r.body); + delete missing[hash]; + return 'done'; + } + } + missing[hash] = { + reason: map ? `map ${map.id} found but version zip unavailable` : 'hash unknown to BeatSaver', + checkedAt: new Date().toISOString() + }; + return 'missing'; + }); + fs.mkdirSync(bs(), { recursive: true }); + save(MISSING_FILE(), JSON.stringify(missing, null, 2) + '\n'); + console.log(` missing.json: ${Object.keys(missing).length} hashes`); + if (res.failed.length) console.warn(` ${res.failed.length} failed; rerun to retry`); +} diff --git a/scripts/archive/cli.ts b/scripts/archive/cli.ts new file mode 100644 index 0000000..e03ab43 --- /dev/null +++ b/scripts/archive/cli.ts @@ -0,0 +1,236 @@ +#!/usr/bin/env node +// plebleader archive CLI. Run from the repo root: node scripts/archive/cli.ts [options] +// +// auth login | refresh | status the sync's own BeatLeader OAuth token chain +// export bl [--since D] BL profile, scores, replays, score stats, leaderboards +// export ss [--since D] ScoreSaber player snapshot and scores (fast) +// export ss-replays [--since D] ScoreSaber replays, ~10.5 s apart (run in background) +// export maps [--maps-since D] BeatSaver JSON + zip for played hashes (default: last 2 years, +// plus all legacy ScoreSaber maps) +// export all [--since D] bl, ss, maps (not ss-replays) +// report [--since D] counts and gaps in the archive +// +// --since: YYYY-MM-DD, unix seconds, or "all" (default: all). --maps-since defaults to the two-year +// window (DEFAULT_MAPS_SINCE). Every command skips files that already exist, so reruns only fill gaps. +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { ARCHIVE_DIR, BL_API, DEFAULT_MAPS_SINCE, LEGACY_PLAYER_ID, MAIN_PLAYER_ID, PLAYER_IDS, TOKENS_FILE, USER_AGENT, loadDotEnv, parseSince } from './config.ts'; +import { getAccessToken, login, readTokens, refresh } from './auth.ts'; +import { archivedScores, blHashes, exportLeaderboards, exportProfile, exportReplays, exportScoreStats, exportScores } from './beatleader.ts'; +import { archivedSSScores, exportSSPlayer, exportSSReplays, exportSSScores, ssHashes } from './scoresaber.ts'; +import { exportMaps } from './beatsaver.ts'; +import { archivePath, exists, listFiles, readState } from './store.ts'; + +function fmt(ms: number): string { + const d = new Date(ms); + const left = ms - Date.now(); + const hours = Math.round(left / 36e5); + return `${d.toISOString()} (${left < 0 ? 'expired' : hours >= 48 ? `${Math.round(hours / 24)} d left` : `${hours} h left`})`; +} + +async function blGet(path: string, token: string): Promise { + const res = await fetch(`${BL_API}${path}`, { + headers: { Authorization: `Bearer ${token}`, 'User-Agent': USER_AGENT } + }); + if (!res.ok) throw new Error(`GET ${path}: HTTP ${res.status}`); + return (await res.json()) as T; +} + +/** + * Phase 1 step 1. The bearer token must belong to pleb, and an unranked map's ratings must be + * readable through /leaderboards/hash/{hash}. + * + * Verified 2026-11-08: BeatLeader only honors bearer tokens on /oauth2/identity, clan management + * and ReeSabers presets. /user and the showAllRatings gate on score lists read the beatleader.com + * cookie session instead, so score lists always come back with unranked ratings nulled. The + * per-hash leaderboard lookup returns full ratings (incl. modifiersRating) even anonymously. + */ +async function verify(): Promise { + const token = await getAccessToken(); + const identity = await blGet<{ id?: string; name?: string }>('/oauth2/identity', token); + const idOk = identity.id === MAIN_PLAYER_ID; + console.log(`GET /oauth2/identity: ${identity.id} (${identity.name}) ${idOk ? 'OK' : `MISMATCH, expected ${MAIN_PLAYER_ID}`}`); + + type ScorePage = { data: { leaderboard: { song: { name: string; hash: string }; difficulty: { difficultyName: string; status: number } } }[] }; + // status 3 = ranked; anything else counts as unranked for this check. + const page = await blGet(`/player/${MAIN_PLAYER_ID}/scores?count=50&page=1&sortBy=date&order=desc&type=unranked`, token); + const unranked = page.data.find((s) => s.leaderboard.difficulty.status !== 3); + if (!unranked) { + console.log('No unranked score found in the latest 50 to check ratings against.'); + return idOk; + } + type HashLookup = { leaderboards: { difficulty: { difficultyName: string; stars: number | null; techRating: number | null } }[] }; + const { song, difficulty } = unranked.leaderboard; + const lookup = await blGet(`/leaderboards/hash/${song.hash}`, token); + const d = lookup.leaderboards.find((l) => l.difficulty.difficultyName === difficulty.difficultyName)?.difficulty; + const ratingsOk = d?.stars != null; + console.log(`Unranked "${song.name}" ${difficulty.difficultyName} via /leaderboards/hash: stars=${d?.stars} tech=${d?.techRating} ${ratingsOk ? 'OK' : 'MISSING'}`); + return idOk && ratingsOk; +} + +function playedHashes(since: number): string[] { + return [...new Set([...blHashes(since), ...PLAYER_IDS.flatMap((id) => ssHashes(id, since))])].sort(); +} + +/** + * Hashes to fetch BeatSaver map files for: played within the maps window, plus — regardless of + * age — the maps of legacy ScoreSaber scores that have an archived replay, so those replays render. + */ +function mapHashes(mapsSince: number): string[] { + const legacyWithReplay = archivedSSScores(LEGACY_PLAYER_ID, 0) + .filter((it) => exists(archivePath('scoresaber', LEGACY_PLAYER_ID, 'replays', `${it.score.id}.dat`))) + .map((it) => it.leaderboard.songHash.toLowerCase()); + return [...new Set([...playedHashes(mapsSince), ...legacyWithReplay])].sort(); +} + +async function exportBL(since: number): Promise { + for (const id of PLAYER_IDS) { + await exportProfile(id); + await exportScores(id, since); + } + // The scores dir holds both accounts' scores (keyed by score id). + const scores = archivedScores(since); + await exportReplays(scores); + await exportScoreStats(scores); + // BL leaderboard JSON (ratings) for every played hash, not just the maps window: it's small + // and the importer needs it for star ratings on old scores. + await exportLeaderboards(blHashes(since)); +} + +async function exportSS(since: number): Promise { + for (const id of PLAYER_IDS) { + await exportSSPlayer(id); + await exportSSScores(id, since); + } + // Hashes only ScoreSaber knows about still need BL leaderboard data (phase 5 merge). + await exportLeaderboards([...new Set(PLAYER_IDS.flatMap((id) => ssHashes(id, since)))].sort()); +} + +function report(since: number, mapsSince: number): void { + const count = (...p: string[]) => listFiles(archivePath(...p)).length; + const day = (t: number) => (t ? new Date(t * 1000).toISOString().slice(0, 10) : 'all time'); + const bl = archivedScores(since); + const blWithReplay = bl.filter((s) => s.replay); + const blReplayMissing = blWithReplay.filter((s) => !exists(archivePath('beatleader', 'replays', decodeURIComponent(new URL(s.replay!).pathname.split('/').pop()!)))); + const statsMissing = bl.filter((s) => !exists(archivePath('beatleader', 'scorestats', `${s.id}.json`))); + const ss = PLAYER_IDS.map((id) => { + const items = archivedSSScores(id, since); + const withReplay = items.filter((it) => it.score.hasReplay); + const missing = withReplay.filter((it) => !exists(archivePath('scoresaber', id, 'replays', `${it.score.id}.dat`))); + return { id, items, withReplay, missing }; + }); + const allHashes = playedHashes(since); + const noLeaderboard = allHashes.filter((h) => listFiles(archivePath('beatleader', 'leaderboards', h)).length === 0); + const hashes = mapHashes(Math.max(since, mapsSince)); + const noZip = hashes.filter((h) => !exists(archivePath('beatsaver', 'zips', `${h}.zip`))); + let missingJson: Record = {}; + try { + missingJson = JSON.parse(fs.readFileSync(archivePath('beatsaver', 'missing.json'), 'utf-8')); + } catch {} + const du = (dir: string): number => { + let total = 0; + const walk = (d: string) => { + for (const e of fs.readdirSync(d, { withFileTypes: true })) { + const p = path.join(d, e.name); + if (e.isDirectory()) walk(p); + else total += fs.statSync(p).size; + } + }; + if (fs.existsSync(dir)) walk(dir); + return total; + }; + + console.log(`Archive: ${ARCHIVE_DIR}`); + console.log(`Window: scores since ${day(since)}, BeatSaver maps since ${day(mapsSince)}`); + console.log(''); + console.log(`BeatLeader scores ${bl.length} in window (${count('beatleader', 'scores')} archived total, both accounts)`); + console.log(`BeatLeader replays ${blWithReplay.length - blReplayMissing.length}/${blWithReplay.length}`); + console.log(`BeatLeader score stats ${bl.length - statsMissing.length}/${bl.length}`); + for (const s of ss) { + console.log(`ScoreSaber ${s.id}: ${s.items.length} scores, replays ${s.withReplay.length - s.missing.length}/${s.withReplay.length}`); + } + console.log(`Played hashes ${allHashes.length}`); + console.log(` BL leaderboard snapshot ${allHashes.length - noLeaderboard.length}/${allHashes.length}`); + console.log(` BeatSaver zip ${hashes.length - noZip.length}/${hashes.length} in maps window (missing.json lists ${Object.keys(missingJson).length})`); + console.log(`Archive size ${(du(ARCHIVE_DIR) / 1e9).toFixed(2)} GB`); + const failures = readState().failures; + if (failures && Object.keys(failures).length) console.log(`Failures from last runs: ${JSON.stringify(failures)}`); +} + +function option(args: string[], name: string): string | undefined { + const i = args.indexOf(name); + return i >= 0 ? args[i + 1] : args.find((a) => a.startsWith(`${name}=`))?.slice(name.length + 1); +} + +async function main(): Promise { + loadDotEnv(); + const args = process.argv.slice(2); + const [cmd, sub] = args; + + if (cmd === 'auth' && sub === 'login') { + const store = await login(); + console.log(`\nLogged in as ${store.player_name} (${store.player_id}). Tokens saved to ${TOKENS_FILE}`); + console.log(`Refresh token valid until about ${fmt(store.refresh_expires_at)}`); + return (await verify()) ? 0 : 1; + } + if (cmd === 'auth' && sub === 'refresh') { + const store = await refresh(); + console.log(`Refreshed. Access token until ${fmt(store.access_expires_at)}; refresh until about ${fmt(store.refresh_expires_at)}`); + return 0; + } + if (cmd === 'auth' && sub === 'status') { + const store = readTokens(); + if (!store) { + console.log(`No tokens at ${TOKENS_FILE}. Run \`auth login\`.`); + return 1; + } + console.log(`Tokens: ${TOKENS_FILE}`); + console.log(`Player: ${store.player_name} (${store.player_id})`); + console.log(`Access: ${fmt(store.access_expires_at)}`); + console.log(`Refresh: ${fmt(store.refresh_expires_at)} (estimated)`); + return (await verify()) ? 0 : 1; + } + + const since = parseSince(option(args, '--since')); + const mapsSince = Math.max(since, parseSince(option(args, '--maps-since'), DEFAULT_MAPS_SINCE)); + if (cmd === 'export') { + console.log(`Archive ${ARCHIVE_DIR}, since ${since ? new Date(since * 1000).toISOString() : 'all time'}\n`); + switch (sub) { + case 'bl': + await exportBL(since); + return 0; + case 'ss': + await exportSS(since); + return 0; + case 'ss-replays': + // Legacy first: its replays are mostly already on disk from ss-scraper. + for (const id of [...PLAYER_IDS].reverse()) await exportSSReplays(id, since); + return 0; + case 'maps': + await exportMaps(mapHashes(mapsSince)); + return 0; + case 'all': + await exportBL(since); + await exportSS(since); + await exportMaps(mapHashes(mapsSince)); + console.log(''); + report(since, mapsSince); + return 0; + } + } + if (cmd === 'report') { + report(since, mapsSince); + return 0; + } + + console.error('Usage: node scripts/archive/cli.ts [--since YYYY-MM-DD|all] [--maps-since YYYY-MM-DD|all]'); + return 2; +} + +main().then( + (code) => process.exit(code), + (err) => { + console.error(err instanceof Error ? err.message : err); + process.exit(1); + } +); diff --git a/scripts/archive/config.ts b/scripts/archive/config.ts new file mode 100644 index 0000000..c85e533 --- /dev/null +++ b/scripts/archive/config.ts @@ -0,0 +1,76 @@ +// Shared configuration for the plebleader archive CLI. +// Runs directly under Node >= 22 (type stripping), no build step. +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +export const USER_AGENT = 'plebsaber-archive/0.1'; +export const BL_API = 'https://api.beatleader.com'; +export const MAIN_PLAYER_ID = '76561199407393962'; +export const LEGACY_PLAYER_ID = '76561197962107242'; + +export const PLAYER_IDS = [MAIN_PLAYER_ID, LEGACY_PLAYER_ID]; + +/** Scores, replays, score stats and profile data: all history by default. */ +export const DEFAULT_SINCE = 'all'; +/** + * BeatSaver map files (JSON + zip) only for maps played in the last two years. Older maps can + * be fetched later with --maps-since all (existing files are skipped). + */ +export const DEFAULT_MAPS_SINCE = '2024-11-08'; + +/** Where ss-scraper already downloaded ScoreSaber replays (/_.dat). */ +export const SS_SCRAPER_REPLAYS = process.env.SS_SCRAPER_REPLAYS ?? path.join(process.env.HOME ?? '', 'ops/beatsaber/ss-scraper/replays'); + +/** Parse --since: YYYY-MM-DD (UTC), unix seconds, or "all". */ +export function parseSince(value: string | undefined, fallback = DEFAULT_SINCE): number { + const v = value ?? fallback; + if (v === 'all') return 0; + if (/^\d+$/.test(v)) return Number(v); + const ms = Date.parse(/^\d{4}-\d{2}-\d{2}$/.test(v) ? `${v}T00:00:00Z` : v); + if (Number.isNaN(ms)) throw new Error(`Invalid --since: ${v}`); + return Math.floor(ms / 1000); +} + +/** + * Root of the plebleader state (archive/, auth/, db/, ...). + * Production: /var/lib/plebleader. Local default: ./.data/plebleader (gitignored). + */ +export const ROOT = path.resolve(process.env.PLEBLEADER_DIR ?? path.join(process.cwd(), '.data', 'plebleader')); +export const ARCHIVE_DIR = path.join(ROOT, 'archive'); +export const AUTH_DIR = path.join(ROOT, 'auth'); +export const TOKENS_FILE = path.join(AUTH_DIR, 'beatleader_tokens.json'); + +/** Load KEY=VALUE (optionally `export KEY=VALUE`) lines from .env without overriding the real environment. */ +export function loadDotEnv(file = path.join(process.cwd(), '.env')): void { + if (!fs.existsSync(file)) return; + for (const line of fs.readFileSync(file, 'utf-8').split('\n')) { + const m = line.match(/^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/); + if (!m) continue; + const [, key, raw] = m; + const value = raw.replace(/^(['"])(.*)\1$/, '$2'); + if (process.env[key] === undefined) process.env[key] = value; + } +} + +export function clientCredentials(): { clientId: string; clientSecret: string } { + const clientId = process.env.BL_CLIENT_ID; + const clientSecret = process.env.BL_CLIENT_SECRET; + if (!clientId || !clientSecret) { + throw new Error('BL_CLIENT_ID and BL_CLIENT_SECRET must be set (environment or .env).'); + } + return { clientId, clientSecret }; +} + +/** Write a file atomically: temp file in the same directory, fsync, rename. */ +export function writeFileAtomic(file: string, data: string | Uint8Array, mode = 0o644): void { + fs.mkdirSync(path.dirname(file), { recursive: true }); + const tmp = `${file}.tmp-${process.pid}-${Date.now()}`; + const fd = fs.openSync(tmp, 'w', mode); + try { + fs.writeSync(fd, typeof data === 'string' ? Buffer.from(data, 'utf-8') : data); + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } + fs.renameSync(tmp, file); +} diff --git a/scripts/archive/http.ts b/scripts/archive/http.ts new file mode 100644 index 0000000..22fe927 --- /dev/null +++ b/scripts/archive/http.ts @@ -0,0 +1,107 @@ +// Rate-limited GET with 429 / 5xx / network retries. Returns raw bytes so callers can +// store responses byte-for-byte. +import { USER_AGENT } from './config.ts'; + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +/** Enforces a minimum interval between request starts. */ +export class Limiter { + private next = 0; + private readonly minIntervalMs: number; + constructor(minIntervalMs: number) { + this.minIntervalMs = minIntervalMs; + } + async wait(): Promise { + const now = Date.now(); + const at = Math.max(now, this.next); + this.next = at + this.minIntervalMs; + if (at > now) await sleep(at - now); + } + /** Push the next slot out, e.g. after a 429. */ + backoff(ms: number): void { + this.next = Math.max(this.next, Date.now() + ms); + } +} + +export const limiters = { + beatleader: new Limiter(250), // ~4 req/s, well under 50 per 10 s + blCdn: new Limiter(250), + scoresaber: new Limiter(1000), + // /api/v2/scores/{id}/replay sends x-ratelimit-limit-long: 360 per hour (checked 2026-11-08; + // short 20 and medium 60/min don't bind). 10.5 s ≈ 343/h. get() also pauses at remaining 0. + scoresaberReplay: new Limiter(10_500), + beatsaver: new Limiter(500), // ~2 req/s + beatsaverCdn: new Limiter(250) +}; + +export type GetResult = { status: number; body: Buffer; headers: Headers; url: string }; + +export class HttpError extends Error { + readonly status: number; + constructor(status: number, message: string) { + super(message); + this.status = status; + } +} + +/** + * GET with retries. Resolves with the final response for 2xx and for statuses listed in + * `accept` (e.g. 404); throws HttpError for anything else once retries are exhausted. + */ +export async function get( + url: string, + opts: { limiter: Limiter; headers?: Record; accept?: number[]; retries?: number; timeoutMs?: number } +): Promise { + const retries = opts.retries ?? 6; + let backoffMs = 2000; + for (let attempt = 0; ; attempt++) { + await opts.limiter.wait(); + let res: Response; + try { + res = await fetch(url, { + headers: { 'User-Agent': USER_AGENT, ...opts.headers }, + signal: AbortSignal.timeout(opts.timeoutMs ?? 120_000) + }); + } catch (err) { + if (attempt >= retries) throw err; + console.warn(` network error on ${shortUrl(url)}: ${(err as Error).message}; retrying in ${backoffMs / 1000}s`); + await sleep(backoffMs); + backoffMs = Math.min(backoffMs * 2, 300_000); + continue; + } + honorRateLimitHeaders(res.headers, opts.limiter, url); + if (res.ok || opts.accept?.includes(res.status)) { + return { status: res.status, body: Buffer.from(await res.arrayBuffer()), headers: res.headers, url: res.url }; + } + const text = (await res.text().catch(() => '')).slice(0, 200); + const retryable = res.status === 429 || res.status >= 500; + if (!retryable || attempt >= retries) { + throw new HttpError(res.status, `GET ${shortUrl(url)}: HTTP ${res.status} ${text}`); + } + const retryAfter = Number(res.headers.get('retry-after')); + const wait = Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter * 1000 : backoffMs; + console.warn(` HTTP ${res.status} on ${shortUrl(url)}; waiting ${Math.round(wait / 1000)}s`); + opts.limiter.backoff(wait); + backoffMs = Math.min(backoffMs * 2, 300_000); + } +} + +/** + * ScoreSaber-style headers: x-ratelimit-remaining-{short,medium,long} and + * x-ratelimit-reset-{...} (seconds until that window resets). When any window is exhausted, + * hold the limiter until it resets instead of running into a 429. + */ +function honorRateLimitHeaders(headers: Headers, limiter: Limiter, url: string): void { + for (const w of ['short', 'medium', 'long']) { + const remaining = headers.get(`x-ratelimit-remaining-${w}`); + const reset = Number(headers.get(`x-ratelimit-reset-${w}`)); + if (remaining !== null && Number(remaining) <= 0 && Number.isFinite(reset) && reset > 0) { + console.warn(` rate limit window "${w}" exhausted on ${shortUrl(url)}; pausing ${reset}s`); + limiter.backoff((reset + 1) * 1000); + } + } +} + +function shortUrl(url: string): string { + return url.split('?')[0]; +} diff --git a/scripts/archive/scoresaber.ts b/scripts/archive/scoresaber.ts new file mode 100644 index 0000000..7a2cf96 --- /dev/null +++ b/scripts/archive/scoresaber.ts @@ -0,0 +1,101 @@ +// ScoreSaber exporter: player snapshot, scores (score + leaderboard items), replays. +// Public API, no auth. Replays are paced at ~10.5 s (360/hour limit) and run as their own command. +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { SS_SCRAPER_REPLAYS } from './config.ts'; +import { get, limiters } from './http.ts'; +import { archivePath, eachWithProgress, exists, listFiles, readJson, save, saveSnapshotIfChanged, updateState } from './store.ts'; + +const SS_API = 'https://scoresaber.com/api'; +const ss = (playerId: string, ...p: string[]) => archivePath('scoresaber', playerId, ...p); + +export type SSItem = { + score: { id: number; timeSet: string; hasReplay: boolean }; + leaderboard: { songHash: string }; +}; + +export function archivedSSScores(playerId: string, sinceUnix = 0): SSItem[] { + const dir = ss(playerId, 'scores'); + return listFiles(dir, '.json') + .map((f) => readJson(path.join(dir, f))) + .filter((it) => Date.parse(it.score.timeSet) / 1000 >= sinceUnix); +} + +export async function exportSSPlayer(playerId: string): Promise { + const r = await get(`${SS_API}/player/${playerId}/full`, { limiter: limiters.scoresaber }); + console.log(`ScoreSaber player ${playerId}: ${saveSnapshotIfChanged(ss(playerId, 'player'), r.body) ? 'saved' : 'unchanged'}`); +} + +/** Page `sort=recent` until scores are older than `sinceUnix`. */ +export async function exportSSScores(playerId: string, sinceUnix: number): Promise { + console.log(`ScoreSaber scores for ${playerId} since ${new Date(sinceUnix * 1000).toISOString()}`); + let added = 0; + let seen = 0; + for (let page = 1; ; page++) { + const r = await get(`${SS_API}/player/${playerId}/scores?limit=100&page=${page}&sort=recent`, { limiter: limiters.scoresaber }); + const json = JSON.parse(r.body.toString('utf-8')) as { playerScores: SSItem[]; metadata: { total: number } }; + let reachedCutoff = false; + for (const item of json.playerScores) { + if (Date.parse(item.score.timeSet) / 1000 < sinceUnix) { + reachedCutoff = true; + break; + } + seen++; + const file = ss(playerId, 'scores', `${item.score.id}.json`); + if (exists(file)) continue; + save(file, JSON.stringify(item)); + added++; + } + console.log(` page ${page}: ${seen} in window, ${added} new (total on server ${json.metadata.total})`); + if (reachedCutoff || json.playerScores.length < 100) break; + } + updateState(`scoresaber:${playerId}`, { scoresSince: sinceUnix, lastScoresRun: new Date().toISOString() }); +} + +/** Download replays for archived scores with hasReplay, oldest first, ~10.5 s apart. */ +export async function exportSSReplays(playerId: string, sinceUnix: number): Promise { + const items = archivedSSScores(playerId, sinceUnix) + .filter((it) => it.score.hasReplay) + .sort((a, b) => Date.parse(a.score.timeSet) - Date.parse(b.score.timeSet)); + seedFromScraper(playerId, items); + const todo = items.filter((it) => !exists(ss(playerId, 'replays', `${it.score.id}.dat`))); + console.log(`ScoreSaber replays for ${playerId}: ${items.length} with hasReplay, ${todo.length} to download (~${Math.ceil((todo.length * 10.5) / 60)} min)`); + const res = await eachWithProgress( + 'ss-replays', + todo, + async (it) => { + const r = await get(`${SS_API}/v2/scores/${it.score.id}/replay`, { + limiter: limiters.scoresaberReplay, + accept: [404], + timeoutMs: 300_000 + }); + if (r.status === 404) return 'missing'; + save(ss(playerId, 'replays', `${it.score.id}.dat`), r.body); + return 'done'; + }, + 10 + ); + if (res.missing || res.failed.length) { + updateState('failures', { [`scoresaber:${playerId}:replays`]: res.failed.map((f) => f.item.score.id) }); + } +} + +/** Copy replays ss-scraper already downloaded (_.dat) instead of fetching them again. */ +function seedFromScraper(playerId: string, items: SSItem[]): void { + const dir = path.join(SS_SCRAPER_REPLAYS, playerId); + const byId = new Map(); + for (const f of listFiles(dir, '.dat')) byId.set(f.split('_')[0], f); + let copied = 0; + for (const it of items) { + const src = byId.get(String(it.score.id)); + const dest = ss(playerId, 'replays', `${it.score.id}.dat`); + if (!src || exists(dest)) continue; + save(dest, fs.readFileSync(path.join(dir, src))); + copied++; + } + if (copied) console.log(` copied ${copied} replays from ${dir}`); +} + +export function ssHashes(playerId: string, sinceUnix = 0): string[] { + return [...new Set(archivedSSScores(playerId, sinceUnix).map((it) => it.leaderboard.songHash.toLowerCase()))].sort(); +} diff --git a/scripts/archive/store.ts b/scripts/archive/store.ts new file mode 100644 index 0000000..de8f410 --- /dev/null +++ b/scripts/archive/store.ts @@ -0,0 +1,80 @@ +// Archive filesystem helpers. The archive is append-only: files are written once +// (atomically) and never rewritten, except state.json. +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { ARCHIVE_DIR, writeFileAtomic } from './config.ts'; + +export const archivePath = (...parts: string[]) => path.join(ARCHIVE_DIR, ...parts); + +export const exists = (file: string) => fs.existsSync(file); + +export function save(file: string, data: string | Uint8Array): void { + writeFileAtomic(file, data); +} + +/** Today's date (UTC) as YYYY-MM-DD, used for dated snapshot filenames. */ +export const today = () => new Date().toISOString().slice(0, 10); + +export function listFiles(dir: string, ext?: string): string[] { + try { + return fs.readdirSync(dir).filter((f) => !f.includes('.tmp-') && (!ext || f.endsWith(ext))); + } catch { + return []; + } +} + +export function readJson(file: string): T { + return JSON.parse(fs.readFileSync(file, 'utf-8')) as T; +} + +/** + * Save a dated snapshot in `dir` only when the content differs from the newest existing one. + * Returns true if a new snapshot was written. + */ +export function saveSnapshotIfChanged(dir: string, body: Buffer): boolean { + const latest = listFiles(dir, '.json').sort().at(-1); + if (latest && fs.readFileSync(path.join(dir, latest)).equals(body)) return false; + save(path.join(dir, `${today()}.json`), body); + return true; +} + +type State = Record>; +const STATE_FILE = () => archivePath('state.json'); + +export function readState(): State { + try { + return readJson(STATE_FILE()); + } catch { + return {}; + } +} + +export function updateState(section: string, patch: Record): void { + const state = readState(); + state[section] = { ...state[section], ...patch }; + writeFileAtomic(STATE_FILE(), JSON.stringify(state, null, 2) + '\n'); +} + +/** Run `fn` over items with a progress line every `every` items. Errors are collected, not thrown. */ +export async function eachWithProgress( + label: string, + items: T[], + fn: (item: T) => Promise<'done' | 'skipped' | 'missing'>, + every = 50 +): Promise<{ done: number; skipped: number; missing: number; failed: { item: T; error: string }[] }> { + const out = { done: 0, skipped: 0, missing: 0, failed: [] as { item: T; error: string }[] }; + let i = 0; + for (const item of items) { + i++; + try { + out[await fn(item)]++; + } catch (err) { + out.failed.push({ item, error: (err as Error).message }); + console.warn(` ${label}: ${(err as Error).message}`); + } + if (i % every === 0 || i === items.length) { + console.log(` ${label}: ${i}/${items.length} (new ${out.done}, existing ${out.skipped}, missing ${out.missing}, failed ${out.failed.length})`); + } + } + return out; +}