plebsaber.stream/scripts/archive/http.ts

// Rate-limited GET with 429 / 5xx / network retries. Returns raw bytes so callers can
// store responses byte-for-byte.
import { USER_AGENT } from './config.ts';

const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));

/** Enforces a minimum interval between request starts. */
export class Limiter {
	private next = 0;
	private readonly minIntervalMs: number;
	constructor(minIntervalMs: number) {
		this.minIntervalMs = minIntervalMs;
	}
	async wait(): Promise<void> {
		const now = Date.now();
		const at = Math.max(now, this.next);
		this.next = at + this.minIntervalMs;
		if (at > now) await sleep(at - now);
	}
	/** Push the next slot out, e.g. after a 429. */
	backoff(ms: number): void {
		this.next = Math.max(this.next, Date.now() + ms);
	}
}

export const limiters = {
	beatleader: new Limiter(250), // ~4 req/s, well under 50 per 10 s
	blCdn: new Limiter(250),
	scoresaber: new Limiter(1000),
	// /api/v2/scores/{id}/replay sends x-ratelimit-limit-long: 360 per hour (checked 2026-11-08;
	// short 20 and medium 60/min don't bind). 10.5 s ≈ 343/h. get() also pauses at remaining 0.
	scoresaberReplay: new Limiter(10_500),
	beatsaver: new Limiter(500), // ~2 req/s
	beatsaverCdn: new Limiter(250)
};

export type GetResult = { status: number; body: Buffer; headers: Headers; url: string };

export class HttpError extends Error {
	readonly status: number;
	constructor(status: number, message: string) {
		super(message);
		this.status = status;
	}
}

/**
 * GET with retries. Resolves with the final response for 2xx and for statuses listed in
 * `accept` (e.g. 404); throws HttpError for anything else once retries are exhausted.
 */
export async function get(
	url: string,
	opts: { limiter: Limiter; headers?: Record<string, string>; accept?: number[]; retries?: number; timeoutMs?: number }
): Promise<GetResult> {
	const retries = opts.retries ?? 6;
	let backoffMs = 2000;
	for (let attempt = 0; ; attempt++) {
		await opts.limiter.wait();
		let res: Response;
		try {
			res = await fetch(url, {
				headers: { 'User-Agent': USER_AGENT, ...opts.headers },
				signal: AbortSignal.timeout(opts.timeoutMs ?? 120_000)
			});
		} catch (err) {
			if (attempt >= retries) throw err;
			console.warn(`  network error on ${shortUrl(url)}: ${(err as Error).message}; retrying in ${backoffMs / 1000}s`);
			await sleep(backoffMs);
			backoffMs = Math.min(backoffMs * 2, 300_000);
			continue;
		}
		honorRateLimitHeaders(res.headers, opts.limiter, url);
		if (res.ok || opts.accept?.includes(res.status)) {
			return { status: res.status, body: Buffer.from(await res.arrayBuffer()), headers: res.headers, url: res.url };
		}
		const text = (await res.text().catch(() => '')).slice(0, 200);
		const retryable = res.status === 429 || res.status >= 500;
		if (!retryable || attempt >= retries) {
			throw new HttpError(res.status, `GET ${shortUrl(url)}: HTTP ${res.status} ${text}`);
		}
		const retryAfter = Number(res.headers.get('retry-after'));
		const wait = Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter * 1000 : backoffMs;
		console.warn(`  HTTP ${res.status} on ${shortUrl(url)}; waiting ${Math.round(wait / 1000)}s`);
		opts.limiter.backoff(wait);
		backoffMs = Math.min(backoffMs * 2, 300_000);
	}
}

/**
 * ScoreSaber-style headers: x-ratelimit-remaining-{short,medium,long} and
 * x-ratelimit-reset-{...} (seconds until that window resets). When any window is exhausted,
 * hold the limiter until it resets instead of running into a 429.
 */
function honorRateLimitHeaders(headers: Headers, limiter: Limiter, url: string): void {
	for (const w of ['short', 'medium', 'long']) {
		const remaining = headers.get(`x-ratelimit-remaining-${w}`);
		const reset = Number(headers.get(`x-ratelimit-reset-${w}`));
		if (remaining !== null && Number(remaining) <= 0 && Number.isFinite(reset) && reset > 0) {
			console.warn(`  rate limit window "${w}" exhausted on ${shortUrl(url)}; pausing ${reset}s`);
			limiter.backoff((reset + 1) * 1000);
		}
	}
}

function shortUrl(url: string): string {
	return url.split('?')[0];
}