1/**2 * Daily sync of the AI-model catalog from https://models.dev/api.json into3 * the `models` table. Attribution tags on scripts reference these ids, so4 * rows are upserted and never deleted — a tag must stay resolvable even if5 * the model is delisted upstream.6 *7 * api.json shape: { <providerId>: { name, models: { <modelId>: {...} } } }.8 * The same model appears under many hosting providers; we dedupe by model id9 * and prefer the entry with the richest metadata (release_date present).10 */11import type { Env } from './env';1213export const MODELS_DEV_URL = 'https://models.dev/api.json';1415export interface FlatModel {16 id: string;17 name: string;18 lab: string;19 openWeights: boolean;20 releaseDate: string | null;21 reasoning: boolean | null;22 lastUpdated: string | null;23}2425interface RawModel {26 id?: string;27 name?: string;28 open_weights?: boolean;29 release_date?: string;30 reasoning?: boolean;31 last_updated?: string;32}3334const DATE_RE = /^\d{4}-\d{2}-\d{2}$/;3536/** Flatten providers -> models, deduping by model id. Exported for tests. */37export function flattenCatalog(raw: unknown): FlatModel[] {38 if (raw === null || typeof raw !== 'object') return [];39 const byId = new Map<string, FlatModel>();40 for (const [providerId, providerRaw] of Object.entries(raw as Record<string, unknown>)) {41 if (providerRaw === null || typeof providerRaw !== 'object') continue;42 const models = (providerRaw as { models?: unknown }).models;43 if (models === null || models === undefined || typeof models !== 'object') continue;44 for (const [key, m] of Object.entries(models as Record<string, RawModel>)) {45 if (m === null || typeof m !== 'object') continue;46 const id = typeof m.id === 'string' && m.id.length > 0 ? m.id : key;47 if (id.length === 0 || id.length > 120) continue;48 const name = typeof m.name === 'string' && m.name.length > 0 ? m.name : id;49 const lab = id.includes('/') ? id.slice(0, id.indexOf('/')) : providerId;50 const flat: FlatModel = {51 id,52 name: name.slice(0, 120),53 lab: lab.slice(0, 64),54 openWeights: m.open_weights === true,55 releaseDate:56 typeof m.release_date === 'string' && DATE_RE.test(m.release_date)57 ? m.release_date58 : null,59 reasoning: typeof m.reasoning === 'boolean' ? m.reasoning : null,60 lastUpdated: typeof m.last_updated === 'string' ? m.last_updated.slice(0, 10) : null,61 };62 const existing = byId.get(id);63 // Prefer the richer entry: release date known beats unknown.64 if (existing === undefined || (existing.releaseDate === null && flat.releaseDate !== null)) {65 byId.set(id, flat);66 }67 }68 }69 return [...byId.values()];70}7172export async function syncModels(env: Env, fetchImpl: typeof fetch = fetch): Promise<number> {73 const res = await fetchImpl(MODELS_DEV_URL, {74 headers: { accept: 'application/json' },75 });76 if (!res.ok) throw new Error(`models.dev fetch failed: ${res.status}`);77 const flat = flattenCatalog(await res.json());78 if (flat.length === 0) return 0; // never wipe on a bad payload79 const now = Math.trunc(Date.now() / 1000);80 const stmt = env.DB.prepare(81 `INSERT INTO models (id, name, lab, open_weights, release_date, reasoning, last_updated, synced_at)82 VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)83 ON CONFLICT(id) DO UPDATE SET84 name = excluded.name, lab = excluded.lab, open_weights = excluded.open_weights,85 release_date = excluded.release_date, reasoning = excluded.reasoning,86 last_updated = excluded.last_updated, synced_at = excluded.synced_at`,87 );88 // D1 batch caps are generous but finite; chunk to stay well inside them.89 const CHUNK = 80;90 for (let i = 0; i < flat.length; i += CHUNK) {91 await env.DB.batch(92 flat.slice(i, i + CHUNK).map((m) =>93 stmt.bind(94 m.id,95 m.name,96 m.lab,97 m.openWeights ? 1 : 0,98 m.releaseDate,99 m.reasoning === null ? null : m.reasoning ? 1 : 0,100 m.lastUpdated,101 now,102 ),103 ),104 );105 }106 return flat.length;107}