plebsaber.stream/scripts/archive/beatleader.ts

// BeatLeader exporter: scores, replays, score stats, leaderboards per played hash, profile.
// Writes only to archive/beatleader/. Everything is resumable: existing files are skipped.
import * as path from 'node:path';
import { BL_API } from './config.ts';
import { get, limiters } from './http.ts';
import { archivePath, eachWithProgress, exists, listFiles, readJson, readState, save, saveSnapshotIfChanged, today, updateState } from './store.ts';

const bl = (...p: string[]) => archivePath('beatleader', ...p);

export type BLScore = { id: number; playerId: string; timepost: number; replay?: string | null; leaderboard: { song: { hash: string } } };

async function blGet(pathAndQuery: string, accept?: number[]) {
	return get(`${BL_API}${pathAndQuery}`, { limiter: limiters.beatleader, accept });
}

/** Archived BL scores (parsed), optionally only those at or after `sinceUnix`. */
export function archivedScores(sinceUnix = 0): BLScore[] {
	const dir = bl('scores');
	return listFiles(dir, '.json')
		.map((f) => readJson<BLScore>(path.join(dir, f)))
		.filter((s) => s.timepost >= sinceUnix);
}

export async function exportProfile(playerId: string): Promise<void> {
	console.log(`BeatLeader profile ${playerId}`);
	const dir = (kind: string) => bl('players', playerId, kind);
	const endpoints: [string, string][] = [
		['profile', `/player/${playerId}?stats=true`],
		// count: days of history; the default is 50, which loses the triangle's
		// monthly points. Upstream keeps one row per day since the first score.
		['history', `/player/${playerId}/history?count=100000`],
		['triangle', `/player/${playerId}/history/triangle`],
		['accgraph', `/player/${playerId}/accgraph`]
	];
	for (const [kind, url] of endpoints) {
		const res = await blGet(url);
		const wrote = saveSnapshotIfChanged(dir(kind), res.body);
		console.log(`  ${kind}: ${wrote ? `saved ${today()}` : 'unchanged'}`);
	}
}

/** Page through best scores newest-first from `sinceUnix`, saving each score object. Returns the new ones. */
export async function exportScores(playerId: string, sinceUnix: number): Promise<BLScore[]> {
	console.log(`BeatLeader scores for ${playerId} since ${new Date(sinceUnix * 1000).toISOString()}`);
	const added: BLScore[] = [];
	let seen = 0;
	let newest = Number(readState()[`beatleader:${playerId}`]?.newestTimepost ?? 0);
	for (let page = 1; ; page++) {
		const qs = `count=100&page=${page}&sortBy=date&order=desc&includeIO=true&time_from=${sinceUnix}`;
		const res = await blGet(`/player/${playerId}/scores?${qs}`);
		const json = JSON.parse(res.body.toString('utf-8')) as { metadata: { total: number }; data: BLScore[] };
		for (const score of json.data) {
			seen++;
			newest = Math.max(newest, score.timepost);
			const file = bl('scores', `${score.id}.json`);
			if (exists(file)) continue;
			// The page is parsed and each element re-serialized: same content, no reshaping.
			save(file, JSON.stringify(score));
			added.push(score);
		}
		console.log(`  page ${page}: ${seen}/${json.metadata.total} seen, ${added.length} new`);
		if (json.data.length < 100 || seen >= json.metadata.total) break;
	}
	updateState(`beatleader:${playerId}`, { scoresSince: sinceUnix, newestTimepost: newest, lastScoresRun: new Date().toISOString() });
	return added;
}

/** Newest archived score time for a player (unix seconds), 0 when there is none. */
export function newestScoreTime(playerId: string): number {
	return archivedScores().reduce((max, s) => (s.playerId === playerId ? Math.max(max, s.timepost) : max), 0);
}

export async function exportReplays(scores: BLScore[]): Promise<void> {
	const withReplay = scores.filter((s) => s.replay);
	console.log(`BeatLeader replays: ${withReplay.length} scores with a replay URL`);
	const res = await eachWithProgress('replays', withReplay, async (s) => {
		const name = decodeURIComponent(new URL(s.replay!).pathname.split('/').pop()!);
		const file = bl('replays', name);
		if (exists(file)) return 'skipped';
		const r = await get(s.replay!, { limiter: limiters.blCdn, accept: [404] });
		if (r.status === 404) return 'missing';
		save(file, r.body);
		return 'done';
	});
	recordFailures('replays', res.failed.map((f) => f.item.id));
}

export async function exportScoreStats(scores: BLScore[]): Promise<void> {
	console.log(`BeatLeader score stats: ${scores.length} scores`);
	const res = await eachWithProgress('scorestats', scores, async (s) => {
		const file = bl('scorestats', `${s.id}.json`);
		if (exists(file)) return 'skipped';
		// 302 to a presigned R2 URL; fetch follows it.
		const r = await blGet(`/score/statistic/${s.id}`, [404]);
		if (r.status === 404) return 'missing';
		save(file, r.body);
		return 'done';
	});
	recordFailures('scorestats', res.failed.map((f) => f.item.id));
}

/** /leaderboards/hash/{hash}: all difficulties with full ratings (also unranked, no auth needed). */
export async function exportLeaderboards(hashes: string[], refresh = false): Promise<void> {
	console.log(`BeatLeader leaderboards: ${hashes.length} hashes${refresh ? ' (refresh)' : ''}`);
	const res = await eachWithProgress('leaderboards', hashes, async (hash) => {
		const dir = bl('leaderboards', hash);
		if (!refresh && listFiles(dir, '.json').length > 0) return 'skipped';
		const r = await blGet(`/leaderboards/hash/${hash}`, [404]);
		if (r.status === 404) return 'missing';
		return saveSnapshotIfChanged(dir, r.body) ? 'done' : 'skipped';
	});
	recordFailures('leaderboards', res.failed.map((f) => f.item));
}

function recordFailures(kind: string, ids: (string | number)[]): void {
	if (ids.length) {
		updateState('failures', { [`beatleader:${kind}`]: ids });
		console.warn(`  ${ids.length} ${kind} failed; rerun to retry`);
	}
}

export function blHashes(sinceUnix = 0): string[] {
	return [...new Set(archivedScores(sinceUnix).map((s) => s.leaderboard.song.hash.toLowerCase()))].sort();
}