// Rate-limited GET with 429 / 5xx / network retries. Returns raw bytes so callers can // store responses byte-for-byte. import { USER_AGENT } from './config.ts'; const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); /** Enforces a minimum interval between request starts. */ export class Limiter { private next = 0; private readonly minIntervalMs: number; constructor(minIntervalMs: number) { this.minIntervalMs = minIntervalMs; } async wait(): Promise { const now = Date.now(); const at = Math.max(now, this.next); this.next = at + this.minIntervalMs; if (at > now) await sleep(at - now); } /** Push the next slot out, e.g. after a 429. */ backoff(ms: number): void { this.next = Math.max(this.next, Date.now() + ms); } } export const limiters = { beatleader: new Limiter(250), // ~4 req/s, well under 50 per 10 s blCdn: new Limiter(250), scoresaber: new Limiter(1000), // /api/v2/scores/{id}/replay sends x-ratelimit-limit-long: 360 per hour (checked 2026-11-08; // short 20 and medium 60/min don't bind). 10.5 s ≈ 343/h. get() also pauses at remaining 0. scoresaberReplay: new Limiter(10_500), beatsaver: new Limiter(500), // ~2 req/s beatsaverCdn: new Limiter(250) }; export type GetResult = { status: number; body: Buffer; headers: Headers; url: string }; export class HttpError extends Error { readonly status: number; constructor(status: number, message: string) { super(message); this.status = status; } } /** * GET with retries. Resolves with the final response for 2xx and for statuses listed in * `accept` (e.g. 404); throws HttpError for anything else once retries are exhausted. */ export async function get( url: string, opts: { limiter: Limiter; headers?: Record; accept?: number[]; retries?: number; timeoutMs?: number } ): Promise { const retries = opts.retries ?? 6; let backoffMs = 2000; for (let attempt = 0; ; attempt++) { await opts.limiter.wait(); let res: Response; try { res = await fetch(url, { headers: { 'User-Agent': USER_AGENT, ...opts.headers }, signal: AbortSignal.timeout(opts.timeoutMs ?? 120_000) }); } catch (err) { if (attempt >= retries) throw err; console.warn(` network error on ${shortUrl(url)}: ${(err as Error).message}; retrying in ${backoffMs / 1000}s`); await sleep(backoffMs); backoffMs = Math.min(backoffMs * 2, 300_000); continue; } honorRateLimitHeaders(res.headers, opts.limiter, url); if (res.ok || opts.accept?.includes(res.status)) { return { status: res.status, body: Buffer.from(await res.arrayBuffer()), headers: res.headers, url: res.url }; } const text = (await res.text().catch(() => '')).slice(0, 200); const retryable = res.status === 429 || res.status >= 500; if (!retryable || attempt >= retries) { throw new HttpError(res.status, `GET ${shortUrl(url)}: HTTP ${res.status} ${text}`); } const retryAfter = Number(res.headers.get('retry-after')); const wait = Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter * 1000 : backoffMs; console.warn(` HTTP ${res.status} on ${shortUrl(url)}; waiting ${Math.round(wait / 1000)}s`); opts.limiter.backoff(wait); backoffMs = Math.min(backoffMs * 2, 300_000); } } /** * ScoreSaber-style headers: x-ratelimit-remaining-{short,medium,long} and * x-ratelimit-reset-{...} (seconds until that window resets). When any window is exhausted, * hold the limiter until it resets instead of running into a 429. */ function honorRateLimitHeaders(headers: Headers, limiter: Limiter, url: string): void { for (const w of ['short', 'medium', 'long']) { const remaining = headers.get(`x-ratelimit-remaining-${w}`); const reset = Number(headers.get(`x-ratelimit-reset-${w}`)); if (remaining !== null && Number(remaining) <= 0 && Number.isFinite(reset) && reset > 0) { console.warn(` rate limit window "${w}" exhausted on ${shortUrl(url)}; pausing ${reset}s`); limiter.backoff((reset + 1) * 1000); } } } function shortUrl(url: string): string { return url.split('?')[0]; }