1/**2 * Generates `llms.txt` and `llms-full.txt` into `website/static/` from the same3 * sources the docs site builds from: `website/docs-manifest.json` for the4 * user docs (order and inclusion shared with `website/sidebars.ts`) and the5 * pages `tools/api_docs.ts` emitted into `website/api-gen/`. Run via6 * `deno task site:llms`, after `deno task site:api`.7 */89const OUT_DIR = "website/static";10const API_DIR = "website/api-gen";1112type ManifestItem = { id: string; label: string };13type ManifestSection = {14 label: string | null;15 indexId?: string;16 items: ManifestItem[];17};18type Manifest = { siteUrl: string; sections: ManifestSection[] };1920const manifest = JSON.parse(21 await Deno.readTextFile("website/docs-manifest.json"),22) as Manifest;23const denoJson = JSON.parse(await Deno.readTextFile("deno.json")) as {24 version: string;25 description: string;26};27const siteUrl = Deno.env.get("SITE_URL") ?? manifest.siteUrl;2829/** `README` ids collapse to their directory index route, matching Docusaurus. */30function docUrl(id: string): string {31 const slug = id === "README" ? "" : id.endsWith("/README") ? id.slice(0, -"/README".length) : id;32 return slug === "" ? `${siteUrl}/docs` : `${siteUrl}/docs/${slug}`;33}3435function stripFrontmatter(markdown: string): string {36 if (!markdown.startsWith("---\n")) return markdown;37 const end = markdown.indexOf("\n---\n", 4);38 return end === -1 ? markdown : markdown.slice(end + "\n---\n".length);39}4041function firstHeading(markdown: string): string | null {42 return stripFrontmatter(markdown).match(/^#\s+(.+)$/m)?.[1]?.trim() ?? null;43}4445type DocEntry = { title: string; url: string; sourcePath: string };46type Section = { label: string; docs: DocEntry[] };4748const sections: Section[] = [];49for (const section of manifest.sections) {50 const docs: DocEntry[] = [];51 const ids = [52 ...(section.indexId ? [{ id: section.indexId, label: section.label ?? "" }] : []),53 ...section.items,54 ];55 for (const item of ids) {56 const sourcePath = `docs/${item.id}.md`;57 const markdown = await Deno.readTextFile(sourcePath);58 docs.push({59 title: firstHeading(markdown) ?? item.label,60 url: docUrl(item.id),61 sourcePath,62 });63 }64 sections.push({ label: section.label ?? "Documentation", docs });65}6667// API pages in sidebar order: index first, then top-level pages by68// sidebar_position, then recipes.69type ApiEntry = {70 title: string;71 url: string;72 sourcePath: string;73 position: number;74};75const apiEntries: ApiEntry[] = [];76async function collectApiDir(dir: string, urlPrefix: string): Promise<void> {77 for await (const entry of Deno.readDir(dir)) {78 if (entry.isDirectory) {79 await collectApiDir(`${dir}/${entry.name}`, `${urlPrefix}/${entry.name}`);80 continue;81 }82 if (!entry.name.endsWith(".md")) continue;83 const sourcePath = `${dir}/${entry.name}`;84 const markdown = await Deno.readTextFile(sourcePath);85 const stem = entry.name.slice(0, -".md".length);86 const position = Number(87 markdown.match(/^sidebar_position:\s*(\d+)$/m)?.[1] ?? 99,88 );89 const url = stem === "index" ? urlPrefix : `${urlPrefix}/${stem}`;90 apiEntries.push({91 title: firstHeading(markdown) ?? stem,92 url,93 sourcePath,94 position: dir === API_DIR ? position : 100 + position,95 });96 }97}98try {99 await collectApiDir(API_DIR, `${siteUrl}/api`);100} catch (error) {101 if (!(error instanceof Deno.errors.NotFound)) throw error;102 console.error(`Missing ${API_DIR}; run \`deno task site:api\` first.`);103 Deno.exit(1);104}105if (apiEntries.length === 0) {106 console.error(107 `No API pages found in ${API_DIR}; run \`deno task site:api\` first.`,108 );109 Deno.exit(1);110}111apiEntries.sort((a, b) => a.position - b.position || a.url.localeCompare(b.url));112113// llms.txt: short index with absolute links, per https://llmstxt.org.114const shortLines: string[] = [115 "# lofi",116 "",117 `> ${denoJson.description}`,118 "",119 `lofi is published on JSR as \`@nzip/lofi\` (v${denoJson.version}).`,120 `This file indexes the documentation deployed at ${siteUrl} for this exact version.`,121 `The full documentation corpus is available at ${siteUrl}/llms-full.txt.`,122 `lofi-node (user-owned sync) is indexed separately at ${siteUrl}/node/llms.txt` +123 ` (full corpus: ${siteUrl}/node/llms-full.txt).`,124 "",125];126for (const section of sections) {127 shortLines.push(`## ${section.label}`, "");128 for (const doc of section.docs) {129 shortLines.push(`- [${doc.title}](${doc.url})`);130 }131 shortLines.push("");132}133shortLines.push("## API reference", "");134for (const entry of apiEntries) {135 shortLines.push(`- [${entry.title}](${entry.url})`);136}137shortLines.push("");138139// llms-full.txt: full concatenation in the same order.140const fullParts: string[] = [141 `# lofi documentation (v${denoJson.version})`,142 "",143 `> ${denoJson.description}`,144 "",145 `Deployed at ${siteUrl}. Package: https://jsr.io/@nzip/lofi`,146 "",147];148const allDocs: DocEntry[] = [149 ...sections.flatMap((section) => section.docs),150 ...apiEntries,151];152for (const doc of allDocs) {153 const markdown = stripFrontmatter(await Deno.readTextFile(doc.sourcePath))154 .trim();155 fullParts.push("---", "", `<!-- Source: ${doc.url} -->`, "", markdown, "");156}157158await Deno.writeTextFile(`${OUT_DIR}/llms.txt`, shortLines.join("\n"));159await Deno.writeTextFile(`${OUT_DIR}/llms-full.txt`, fullParts.join("\n"));160const mainApiCount = apiEntries.length;161162// The node-scoped pair: agents enrolling an app against a self-hosted node163// should not have to ingest the full framework corpus, and vice versa. Pages164// come from the assembled /node/docs set (tools/node_docs.ts) so contract165// pages rendered from the lofi-node checkout are included verbatim.166const NODE_DOCS_DIR = "website/node-docs-gen";167const NODE_API_DIR = "website/node-api-gen";168169const nodeManifest = JSON.parse(170 await Deno.readTextFile("website/node-docs-manifest.json"),171) as Manifest;172173function nodeDocUrl(id: string): string {174 return id === "README" ? `${siteUrl}/node/docs` : `${siteUrl}/node/docs/${id}`;175}176177const nodeSections: Section[] = [];178for (const section of nodeManifest.sections) {179 const docs: DocEntry[] = [];180 for (const item of section.items) {181 const sourcePath = `${NODE_DOCS_DIR}/${item.id}.md`;182 let markdown: string;183 try {184 markdown = await Deno.readTextFile(sourcePath);185 } catch {186 console.error(187 `Missing ${sourcePath}; run \`deno task site:node\` first.`,188 );189 Deno.exit(1);190 }191 docs.push({192 title: firstHeading(markdown) ?? item.label,193 url: nodeDocUrl(item.id),194 sourcePath,195 });196 }197 nodeSections.push({ label: section.label ?? "Documentation", docs });198}199200apiEntries.length = 0;201try {202 await collectApiDir(NODE_API_DIR, `${siteUrl}/node/api`);203} catch (error) {204 if (!(error instanceof Deno.errors.NotFound)) throw error;205 console.error(206 `Missing ${NODE_API_DIR}; run \`deno task site:api:node\` first.`,207 );208 Deno.exit(1);209}210const nodeApiEntries = [...apiEntries];211nodeApiEntries.sort((a, b) => a.position - b.position || a.url.localeCompare(b.url));212213const nodeShort: string[] = [214 "# lofi-node",215 "",216 "> The node a lofi-app user runs to own where their data syncs: one daemon embedding a Jazz sync server, iroh node-to-node transport, and ticket-gated access.",217 "",218 `This file indexes the lofi-node documentation deployed at ${siteUrl}/node.`,219 `The full corpus is available at ${siteUrl}/node/llms-full.txt; the lofi framework itself is indexed at ${siteUrl}/llms.txt.`,220 "",221];222for (const section of nodeSections) {223 nodeShort.push(`## ${section.label}`, "");224 for (const doc of section.docs) {225 nodeShort.push(`- [${doc.title}](${doc.url})`);226 }227 nodeShort.push("");228}229nodeShort.push("## API reference", "");230for (const entry of nodeApiEntries) {231 nodeShort.push(`- [${entry.title}](${entry.url})`);232}233nodeShort.push("");234235const nodeFull: string[] = [236 "# lofi-node documentation",237 "",238 `> Deployed at ${siteUrl}/node. Source: https://github.com/FelineStateMachine/lofi-node`,239 "",240];241for (242 const doc of [243 ...nodeSections.flatMap((section) => section.docs),244 ...nodeApiEntries,245 ]246) {247 const markdown = stripFrontmatter(await Deno.readTextFile(doc.sourcePath))248 .trim();249 nodeFull.push("---", "", `<!-- Source: ${doc.url} -->`, "", markdown, "");250}251252await Deno.mkdir(`${OUT_DIR}/node`, { recursive: true });253await Deno.writeTextFile(`${OUT_DIR}/node/llms.txt`, nodeShort.join("\n"));254await Deno.writeTextFile(`${OUT_DIR}/node/llms-full.txt`, nodeFull.join("\n"));255256const docCount = sections.reduce(257 (sum, section) => sum + section.docs.length,258 0,259);260const nodeDocCount = nodeSections.reduce(261 (sum, section) => sum + section.docs.length,262 0,263);264console.log(265 `Generated llms.txt (${docCount} docs + ${mainApiCount} API pages), node/llms.txt ` +266 `(${nodeDocCount} docs + ${nodeApiEntries.length} API pages), and the -full pair into ${OUT_DIR}/.`,267);