// BeatLeader exporter: scores, replays, score stats, leaderboards per played hash, profile. // Writes only to archive/beatleader/. Everything is resumable: existing files are skipped. import * as path from 'node:path'; import { BL_API } from './config.ts'; import { get, limiters } from './http.ts'; import { archivePath, eachWithProgress, exists, listFiles, readJson, save, saveSnapshotIfChanged, today, updateState } from './store.ts'; const bl = (...p: string[]) => archivePath('beatleader', ...p); type BLScore = { id: number; timepost: number; replay?: string | null; leaderboard: { song: { hash: string } } }; async function blGet(pathAndQuery: string, accept?: number[]) { return get(`${BL_API}${pathAndQuery}`, { limiter: limiters.beatleader, accept }); } /** Archived BL scores (parsed), optionally only those at or after `sinceUnix`. */ export function archivedScores(sinceUnix = 0): BLScore[] { const dir = bl('scores'); return listFiles(dir, '.json') .map((f) => readJson(path.join(dir, f))) .filter((s) => s.timepost >= sinceUnix); } export async function exportProfile(playerId: string): Promise { console.log(`BeatLeader profile ${playerId}`); const dir = (kind: string) => bl('players', playerId, kind); const endpoints: [string, string][] = [ ['profile', `/player/${playerId}?stats=true`], // count: days of history; the default is 50, which loses the triangle's // monthly points. Upstream keeps one row per day since the first score. ['history', `/player/${playerId}/history?count=100000`], ['triangle', `/player/${playerId}/history/triangle`], ['accgraph', `/player/${playerId}/accgraph`] ]; for (const [kind, url] of endpoints) { const res = await blGet(url); const wrote = saveSnapshotIfChanged(dir(kind), res.body); console.log(` ${kind}: ${wrote ? `saved ${today()}` : 'unchanged'}`); } } /** Page through best scores newest-first from `sinceUnix`, saving each score object. */ export async function exportScores(playerId: string, sinceUnix: number): Promise { console.log(`BeatLeader scores for ${playerId} since ${new Date(sinceUnix * 1000).toISOString()}`); let added = 0; let seen = 0; let newest = 0; for (let page = 1; ; page++) { const qs = `count=100&page=${page}&sortBy=date&order=desc&includeIO=true&time_from=${sinceUnix}`; const res = await blGet(`/player/${playerId}/scores?${qs}`); const json = JSON.parse(res.body.toString('utf-8')) as { metadata: { total: number }; data: BLScore[] }; for (const score of json.data) { seen++; newest = Math.max(newest, score.timepost); const file = bl('scores', `${score.id}.json`); if (exists(file)) continue; // The page is parsed and each element re-serialized: same content, no reshaping. save(file, JSON.stringify(score)); added++; } console.log(` page ${page}: ${seen}/${json.metadata.total} seen, ${added} new`); if (json.data.length < 100 || seen >= json.metadata.total) break; } updateState(`beatleader:${playerId}`, { scoresSince: sinceUnix, newestTimepost: newest, lastScoresRun: new Date().toISOString() }); } export async function exportReplays(scores: BLScore[]): Promise { const withReplay = scores.filter((s) => s.replay); console.log(`BeatLeader replays: ${withReplay.length} scores with a replay URL`); const res = await eachWithProgress('replays', withReplay, async (s) => { const name = decodeURIComponent(new URL(s.replay!).pathname.split('/').pop()!); const file = bl('replays', name); if (exists(file)) return 'skipped'; const r = await get(s.replay!, { limiter: limiters.blCdn, accept: [404] }); if (r.status === 404) return 'missing'; save(file, r.body); return 'done'; }); recordFailures('replays', res.failed.map((f) => f.item.id)); } export async function exportScoreStats(scores: BLScore[]): Promise { console.log(`BeatLeader score stats: ${scores.length} scores`); const res = await eachWithProgress('scorestats', scores, async (s) => { const file = bl('scorestats', `${s.id}.json`); if (exists(file)) return 'skipped'; // 302 to a presigned R2 URL; fetch follows it. const r = await blGet(`/score/statistic/${s.id}`, [404]); if (r.status === 404) return 'missing'; save(file, r.body); return 'done'; }); recordFailures('scorestats', res.failed.map((f) => f.item.id)); } /** /leaderboards/hash/{hash}: all difficulties with full ratings (also unranked, no auth needed). */ export async function exportLeaderboards(hashes: string[], refresh = false): Promise { console.log(`BeatLeader leaderboards: ${hashes.length} hashes${refresh ? ' (refresh)' : ''}`); const res = await eachWithProgress('leaderboards', hashes, async (hash) => { const dir = bl('leaderboards', hash); if (!refresh && listFiles(dir, '.json').length > 0) return 'skipped'; const r = await blGet(`/leaderboards/hash/${hash}`, [404]); if (r.status === 404) return 'missing'; return saveSnapshotIfChanged(dir, r.body) ? 'done' : 'skipped'; }); recordFailures('leaderboards', res.failed.map((f) => f.item)); } function recordFailures(kind: string, ids: (string | number)[]): void { if (ids.length) { updateState('failures', { [`beatleader:${kind}`]: ids }); console.warn(` ${ids.length} ${kind} failed; rerun to retry`); } } export function blHashes(sinceUnix = 0): string[] { return [...new Set(archivedScores(sinceUnix).map((s) => s.leaderboard.song.hash.toLowerCase()))].sort(); }