// Archive filesystem helpers. The archive is append-only: files are written once // (atomically) and never rewritten, except state.json. import * as fs from 'node:fs'; import * as path from 'node:path'; import { ARCHIVE_DIR, writeFileAtomic } from './config.ts'; export const archivePath = (...parts: string[]) => path.join(ARCHIVE_DIR, ...parts); export const exists = (file: string) => fs.existsSync(file); export function save(file: string, data: string | Uint8Array): void { writeFileAtomic(file, data); } /** Today's date (UTC) as YYYY-MM-DD, used for dated snapshot filenames. */ export const today = () => new Date().toISOString().slice(0, 10); export function listFiles(dir: string, ext?: string): string[] { try { return fs.readdirSync(dir).filter((f) => !f.includes('.tmp-') && (!ext || f.endsWith(ext))); } catch { return []; } } export function readJson(file: string): T { return JSON.parse(fs.readFileSync(file, 'utf-8')) as T; } /** * Save a dated snapshot in `dir` only when the content differs from the newest existing one. * Returns true if a new snapshot was written. */ export function saveSnapshotIfChanged(dir: string, body: Buffer): boolean { const latest = listFiles(dir, '.json').sort().at(-1); if (latest && fs.readFileSync(path.join(dir, latest)).equals(body)) return false; save(path.join(dir, `${today()}.json`), body); return true; } type State = Record>; const STATE_FILE = () => archivePath('state.json'); export function readState(): State { try { return readJson(STATE_FILE()); } catch { return {}; } } export function updateState(section: string, patch: Record): void { const state = readState(); state[section] = { ...state[section], ...patch }; writeFileAtomic(STATE_FILE(), JSON.stringify(state, null, 2) + '\n'); } /** Run `fn` over items with a progress line every `every` items. Errors are collected, not thrown. */ export async function eachWithProgress( label: string, items: T[], fn: (item: T) => Promise<'done' | 'skipped' | 'missing'>, every = 50 ): Promise<{ done: number; skipped: number; missing: number; failed: { item: T; error: string }[] }> { const out = { done: 0, skipped: 0, missing: 0, failed: [] as { item: T; error: string }[] }; let i = 0; for (const item of items) { i++; try { out[await fn(item)]++; } catch (err) { out.failed.push({ item, error: (err as Error).message }); console.warn(` ${label}: ${(err as Error).message}`); } if (i % every === 0 || i === items.length) { console.log(` ${label}: ${i}/${items.length} (new ${out.done}, existing ${out.skipped}, missing ${out.missing}, failed ${out.failed.length})`); } } return out; }