feat(llms-txt): write each page's Markdown and llms-full.txt

.changeset/llms-full.md

@@ -0,0 +1,6 @@
+---
+'@doc-kit/core': minor
+'@doc-kit/generator-react': minor
+---
+
+Add the `llms-txt-full` generator, writing a `llms-full.txt` file holding the Markdown of every page, each preceded by its URL.

README.md

@@ -72,9 +72,9 @@ Options:
                                (json, json-all, json-simple, legacy-html,
                                legacy-html-all, man-page, legacy-json,
                                legacy-json-all, addon-verify, api-links,
-                               orama-db, llms-txt, sitemap, html,
-                               section-pages) or an import specifier for a
-                               custom generator
+                               orama-db, llms-txt, llms-txt-full, sitemap,
+                               html, section-pages) or an import specifier
+                               for a custom generator
   --ignore <patterns...>       Ignore file patterns (glob)
   -o, --output <directory>     The output directory
   -p, --threads <number>       Number of threads to use (minimum: 1)

docs/generators.md

@@ -15,6 +15,7 @@ npx @doc-kit/cli generate -t html -t orama-db -t sitemap -i "docs/**/*.md" -o ou
 | [`html`](./generators/html.md)                   | The modern documentation site: server-rendered, hydrated, themeable. |
 | [`orama-db`](./generators/orama-db.md)           | The search index behind the `html` site's search box.                |
 | [`llms-txt`](./generators/llms-txt.md)           | An [`llms.txt`](https://llmstxt.org/) index for language models.     |
+| [`llms-txt-full`](./generators/llms-txt-full.md) | Every page's Markdown in one `llms-full.txt`, for language models.   |
 | [`sitemap`](./generators/sitemap.md)             | A `sitemap.xml` for search engines.                                  |
 | [`section-pages`](./generators/section-pages.md) | The `html` site and sitemap, plus one page per section of a module.  |
 

packages/core/README.md

@@ -15,7 +15,7 @@ debugging-only `json-simple`); the rest come from companion packages:
 
 | Package                                                                                | Generators                                                                            |
 | -------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------- |
-| [`@doc-kit/generator-react`](https://www.npmjs.com/package/@doc-kit/generator-react)   | `html` (the modern site), `orama-db`, `llms-txt`, `sitemap`                           |
+| [`@doc-kit/generator-react`](https://www.npmjs.com/package/@doc-kit/generator-react)   | `html` (the modern site), `orama-db`, `llms-txt`, `llms-txt-full`, `sitemap`          |
 | [`@node-core/doc-kit-legacy`](https://www.npmjs.com/package/@node-core/doc-kit-legacy) | `legacy-html`, `legacy-html-all`, `legacy-json`, `legacy-json-all` (Node.js-specific) |
 | [`@node-core/doc-kit`](https://www.npmjs.com/package/@node-core/doc-kit)               | `man-page`, `api-links`, `addon-verify` (Node.js-specific)                            |
 

packages/core/src/generators/index.mjs

@@ -22,6 +22,7 @@ export const publicGenerators = {
   'api-links': '@node-core/doc-kit/api-links',
   'orama-db': '@doc-kit/generator-react/orama-db',
   'llms-txt': '@doc-kit/generator-react/llms-txt',
+  'llms-txt-full': '@doc-kit/generator-react/llms-txt-full',
   sitemap: '@doc-kit/generator-react/sitemap',
   html: '@doc-kit/generator-react/html',
   'section-pages': '@doc-kit/generator-react/section-pages',

packages/react/README.md

@@ -17,6 +17,7 @@ npm install --save-dev @doc-kit/core @doc-kit/generator-react
 | `html`          | The full documentation site — server-rendered pages hydrated with Preact, bundled with Vite. |
 | `orama-db`      | An [Orama](https://orama.com) search index, consumed by the `html` site's search box.        |
 | `llms-txt`      | An [`llms.txt`](https://llmstxt.org/) index for Large Language Models.                       |
+| `llms-txt-full` | Every page's Markdown in one `llms-full.txt`, for Large Language Models.                     |
 | `sitemap`       | A `sitemap.xml` for search engines.                                                          |
 | `section-pages` | The `html` site and sitemap, plus one page per section of every module.                      |
 

packages/react/package.json

@@ -2,7 +2,7 @@
   "name": "@doc-kit/generator-react",
   "type": "module",
   "version": "0.4.1",
-  "description": "React/JSX-based generators for @doc-kit/core: html, jsx-ast, llms-txt, sitemap, and orama-db",
+  "description": "React/JSX-based generators for @doc-kit/core: html, jsx-ast, llms-txt, llms-txt-full, sitemap, and orama-db",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/nodejs/doc-kit.git",
@@ -16,6 +16,7 @@
     "./html/bundlers/vite": "./src/html/bundlers/vite.mjs",
     "./jsx-ast": "./src/jsx-ast/index.mjs",
     "./llms-txt": "./src/llms-txt/index.mjs",
+    "./llms-txt-full": "./src/llms-txt-full/index.mjs",
     "./orama-db": "./src/orama-db/index.mjs",
     "./sitemap": "./src/sitemap/index.mjs",
     "./package.json": "./package.json"

packages/react/src/llms-txt-full/README.md

@@ -0,0 +1,9 @@
+# `llms-txt-full` Generator
+
+The `llms-txt-full` generator creates a `llms-full.txt` file holding the Markdown of every page, each preceded by its URL, so Large Language Models (LLMs) can read the whole documentation in one request. Pages are rebuilt from their metadata entries, leaving out the ones other generators create.
+
+## Configuring
+
+- `output` {string} The directory where `llms-full.txt` will be written.
+- `pageURL` {string} URL template for the URL preceding each page.
+  **Default:** `'{baseURL}{path}.md'`.

packages/react/src/llms-txt-full/__tests__/generate.test.mjs

@@ -0,0 +1,29 @@
+import assert from 'node:assert/strict';
+import { describe, it } from 'node:test';
+
+import { setConfig } from '@doc-kit/core/utils/configuration/index.mjs';
+import { u } from 'unist-builder';
+
+import { generate } from '../generate.mjs';
+
+const config = await setConfig({ target: ['llms-txt-full'] });
+
+const entry = (path, text) => ({
+  path,
+  content: u('root', [u('paragraph', [u('text', text)])]),
+});
+
+describe('llms-txt-full', () => {
+  it('precedes the Markdown of every page with its URL', async () => {
+    config['llms-txt-full'].baseURL = 'https://example.com';
+    config['llms-txt-full'].output = undefined;
+
+    const full = await generate([entry('/a', 'First.'), entry('/b', 'Other.')]);
+
+    assert.equal(
+      full,
+      '---\nurl: https://example.com/a.md\n---\nFirst.\n\n' +
+        '---\nurl: https://example.com/b.md\n---\nOther.\n'
+    );
+  });
+});

packages/react/src/llms-txt-full/generate.mjs

@@ -0,0 +1,31 @@
+'use strict';
+
+import { join } from 'node:path';
+
+import getConfig from '@doc-kit/core/utils/configuration/index.mjs';
+import { populate } from '@doc-kit/core/utils/configuration/templates.mjs';
+import { writeFile } from '@doc-kit/core/utils/file.mjs';
+
+import { buildPages } from './utils/buildPages.mjs';
+
+/**
+ * Generates a llms-full.txt file
+ *
+ * @type {import('./types').Generator['generate']}
+ */
+export async function generate(input) {
+  const config = getConfig('llms-txt-full');
+
+  const full = buildPages(input)
+    .map(
+      ({ path, markdown }) =>
+        `---\nurl: ${populate(config.pageURL, { ...config, path })}\n---\n${markdown}`
+    )
+    .join('\n');
+
+  if (config.output) {
+    await writeFile(join(config.output, 'llms-full.txt'), full);
+  }
+
+  return full;
+}

packages/react/src/llms-txt-full/index.mjs

@@ -0,0 +1,25 @@
+'use strict';
+
+import llmsTxt from '../llms-txt/index.mjs';
+import { generate } from './generate.mjs';
+
+/**
+ * This generator generates a llms-full.txt file holding the Markdown of every
+ * page, for LLMs to read the whole documentation in one request
+ *
+ * @type {import('./types').Generator}
+ */
+export default {
+  name: 'llms-txt-full',
+
+  description:
+    'Generates a llms-full.txt file holding the Markdown of every page, each preceded by its URL',
+
+  dependsOn: '@doc-kit/core/metadata',
+
+  defaultConfiguration: {
+    pageURL: llmsTxt.defaultConfiguration.pageURL,
+  },
+
+  generate,
+};

packages/react/src/llms-txt-full/types.d.ts

@@ -0,0 +1,8 @@
+import { MetadataEntry } from '@doc-kit/core/generators/metadata/types';
+
+export type Generator = GeneratorMetadata<
+  {
+    pageURL: string;
+  },
+  Generate<Array<MetadataEntry>, Promise<string>>
+>;

packages/react/src/llms-txt-full/utils/__tests__/buildPages.test.mjs

@@ -0,0 +1,34 @@
+import assert from 'node:assert/strict';
+import { describe, it } from 'node:test';
+
+import { u } from 'unist-builder';
+
+import { buildPages } from '../buildPages.mjs';
+
+const entry = (path, text, extra = {}) => ({
+  path,
+  content: u('root', [u('paragraph', [u('text', text)])]),
+  ...extra,
+});
+
+describe('buildPages', () => {
+  it('joins the content of the entries of each page, in order', () => {
+    const pages = buildPages([
+      entry('/a', 'First.'),
+      entry('/b', 'Other.'),
+      entry('/a', 'Second.'),
+    ]);
+
+    assert.deepEqual(pages, [
+      { path: '/a', markdown: 'First.\n\nSecond.\n' },
+      { path: '/b', markdown: 'Other.\n' },
+    ]);
+  });
+
+  it('leaves out pages generated by other generators', () => {
+    assert.deepEqual(
+      buildPages([entry('/all', 'All.', { synthetic: true })]),
+      []
+    );
+  });
+});

packages/react/src/llms-txt-full/utils/buildPages.mjs

@@ -0,0 +1,30 @@
+import { getRemark, getRemarkMdx } from '@doc-kit/core/utils/remark.mjs';
+
+/**
+ * The Markdown of every page, from the content of its entries, in the order
+ * of the input.
+ *
+ * @param {Array<import('@doc-kit/core/generators/metadata/types').MetadataEntry>} entries
+ * @returns {Array<{ path: string, markdown: string }>}
+ */
+export const buildPages = entries => {
+  /** @type {Map<string, Array<import('@doc-kit/core/generators/metadata/types').MetadataEntry>>} */
+  const pages = new Map();
+
+  for (const entry of entries) {
+    if (!entry.synthetic) {
+      pages.set(entry.path, [...(pages.get(entry.path) ?? []), entry]);
+    }
+  }
+
+  return [...pages].map(([path, sections]) => {
+    const remark = sections[0].mdx ? getRemarkMdx() : getRemark();
+
+    const markdown = sections
+      .map(({ content }) => remark.stringify(content).trim())
+      .filter(Boolean)
+      .join('\n\n');
+
+    return { path, markdown: `${markdown}\n` };
+  });
+};

packages/react/src/llms-txt/README.md

@@ -8,4 +8,4 @@ The `llms-txt` generator creates a `llms.txt` file to provide information to Lar
 - `templatePath` {string} Path to the template file.
   **Default:** `'template.txt'`.
 - `pageURL` {string} URL template for documentation page links.
-  **Default:** `'{baseURL}/latest/api{path}.md'`.
+  **Default:** `'{baseURL}{path}.md'`.

www/doc-kit.config.mjs

@@ -16,6 +16,7 @@ const PUBLIC_GENERATORS = [
   'html',
   'orama-db',
   'llms-txt',
+  'llms-txt-full',
   'sitemap',
   'section-pages',
   'json',