diff --git a/lib/html-to-markdown.js b/lib/html-to-markdown.js
new file mode 100644
index 000000000..783e82a6d
--- /dev/null
+++ b/lib/html-to-markdown.js
@@ -0,0 +1,155 @@
+// Converts the HTML Asciidoctor renders for a page body into Markdown.
+//
+// Asciidoctor lays out admonitions and callout lists as tables, so without the
+// rules below they would come out as Markdown tables with an empty icon column.
+
+// Domino is a browser DOM in Node. So it can parse HTML into a tree of elements.
+const domino = require('@mixmark-io/domino')
+// Turndown converts a DOM tree of elements into Markdown
+const TurndownService = require('turndown')
+// Lets turndown write HTML tables as GitHub-style Markdown tables
+const { tables } = require('@joplin/turndown-plugin-gfm')
+
+const ADMONITION_LABELS = {
+ note: 'Note',
+ tip: 'Tip',
+ important: 'Important',
+ warning: 'Warning',
+ caution: 'Caution',
+}
+
+function hasClass (node, name) {
+ return node.nodeType === 1 && (node.getAttribute('class') || '').split(/\s+/).includes(name)
+}
+
+// domino, the DOM turndown runs on in Node, returns lists that are not iterable.
+function elements (node, tag) {
+ return Array.from(node.getElementsByTagName(tag))
+}
+
+function quote (text) {
+ return text
+ .trim()
+ .split('\n')
+ .map((line) => (line ? `> ${line}` : '>'))
+ .join('\n')
+}
+
+function fence (text, info = '') {
+ const longestRun = Math.max(0, ...(text.match(/`+/g) || []).map((run) => run.length))
+ const marker = '`'.repeat(Math.max(3, longestRun + 1))
+ return `\n\n${marker}${info}\n${text.replace(/\n$/, '')}\n${marker}\n\n`
+}
+
+// Rewrites the DOM before turndown sees it. Links and images become absolute,
+// because llms-full.txt has no directory to resolve relative ones from.
+// Links to an indexed page go to its Markdown version, without the fragment,
+// because the Markdown has no anchor IDs.
+// The Mermaid source becomes a pre, because that is the only element where
+// turndown keeps whitespace. The
s in its node labels are parsed as
+// elements, so they are turned back into text.
+function prepare (html, pageUrl, markdownUrls) {
+ const document = domino.createDocument(`
${html}`)
+ const base = /^https?:/.test(pageUrl || '') ? pageUrl : undefined
+ const absolute = (url) => (!base || /^[a-z][a-z0-9+.-]*:/i.test(url) ? url : new URL(url, base).href)
+ for (const [tag, attribute] of [['a', 'href'], ['img', 'src']]) {
+ for (const node of elements(document.body, tag)) {
+ const value = node.getAttribute(attribute)
+ if (value) node.setAttribute(attribute, absolute(value))
+ }
+ }
+ for (const node of elements(document.body, 'a')) {
+ const markdownUrl = markdownUrls.get((node.getAttribute('href') || '').split('#')[0])
+ if (markdownUrl) node.setAttribute('href', markdownUrl)
+ }
+ for (const node of elements(document.body, 'div').filter((div) => hasClass(div, 'mermaid'))) {
+ for (const br of elements(node, 'br')) br.parentNode.replaceChild(document.createTextNode('
'), br)
+ const pre = document.createElement('pre')
+ pre.setAttribute('class', 'mermaid')
+ pre.textContent = node.textContent
+ node.parentNode.replaceChild(pre, node)
+ }
+ return document.body
+}
+
+// This is the entry point into the file.
+// It creates a TurndownService which walks a DOM element tree and writes Markdown.
+// `` becomes `##` and so on.
+// We have to add a bunch of rules and plugins which know the details of the
+// Asciidoctor output and the HTML it creates.
+function createConverter (markdownUrls = new Map()) {
+ const turndown = new TurndownService({
+ headingStyle: 'atx',
+ codeBlockStyle: 'fenced',
+ bulletListMarker: '-',
+ emDelimiter: '_',
+ })
+ turndown.use(tables)
+ turndown.remove(['style', 'script'])
+
+ // Section anchors are empty links that only show a CSS glyph. This needs a rule
+ // instead of remove(), because turndown checks remove() after its link rule.
+ turndown.addRule('sectionAnchor', {
+ filter: (node) => node.nodeName === 'A' && hasClass(node, 'anchor'),
+ replacement: () => '',
+ })
+
+ // Asciidoctor puts the language in data-lang. The class that turndown reads by
+ // default says "language-none" on untagged blocks. Literal blocks are a bare pre.
+ turndown.addRule('codeBlock', {
+ filter: (node) => node.nodeName === 'PRE',
+ replacement: (_content, node) => {
+ if (hasClass(node, 'mermaid')) return fence(node.textContent.trim(), 'mermaid')
+ const code = node.firstChild && node.firstChild.nodeName === 'CODE' ? node.firstChild : node
+ return fence(code.textContent, code.getAttribute('data-lang') || '')
+ },
+ })
+
+ // Asciidoctor renders a tab set as a list of labels followed by the panels.
+ // Without its label in front of each panel, the alternatives read like steps.
+ turndown.addRule('tabs', {
+ filter: (node) => node.nodeName === 'DIV' && hasClass(node, 'tabs'),
+ replacement: (_content, node) => {
+ const labels = new Map(elements(node, 'li').map((li) => [li.getAttribute('id'), li.textContent.trim()]))
+ const panels = elements(node, 'div').filter(
+ (div) => hasClass(div, 'tabpanel') && div.parentNode.parentNode === node
+ )
+ const sections = panels.map((panel) => {
+ const label = labels.get(panel.getAttribute('aria-labelledby')) || ''
+ return `**Tab: ${label}**\n\n${turndown.turndown(panel.innerHTML)}`
+ })
+ return `\n\n${sections.join('\n\n')}\n\n`
+ },
+ })
+
+ turndown.addRule('admonition', {
+ filter: (node) => node.nodeName === 'DIV' && hasClass(node, 'admonitionblock'),
+ replacement: (_content, node) => {
+ const type = Object.keys(ADMONITION_LABELS).find((name) => hasClass(node, name))
+ const cell = elements(node, 'td').find((td) => hasClass(td, 'content'))
+ const body = turndown.turndown(cell.innerHTML)
+ return `\n\n${quote(`**${ADMONITION_LABELS[type]}:** ${body}`)}\n\n`
+ },
+ })
+
+ turndown.addRule('calloutList', {
+ filter: (node) => node.nodeName === 'DIV' && hasClass(node, 'colist'),
+ replacement: (_content, node) => {
+ const rows = elements(node, 'tr').map((row) => {
+ const [marker, text] = elements(row, 'td')
+ return `(${marker.textContent.trim()}) ${turndown.turndown(text.innerHTML).replace(/\n+/g, ' ')}`
+ })
+ return `\n\n${rows.join('\n')}\n\n`
+ },
+ })
+
+ turndown.addRule('blockTitle', {
+ filter: (node) => node.nodeName === 'DIV' && hasClass(node, 'title'),
+ replacement: (content) => `\n\n**${content.trim()}**\n\n`,
+ })
+
+ // Paragraphs inside list items leave whitespace-only lines behind.
+ return (html, pageUrl) => turndown.turndown(prepare(html, pageUrl, markdownUrls)).replace(/[ \t]+$/gm, '')
+}
+
+module.exports = { createConverter }
diff --git a/lib/llms-txt.js b/lib/llms-txt.js
index 274db7e4e..9fe412767 100644
--- a/lib/llms-txt.js
+++ b/lib/llms-txt.js
@@ -1,21 +1,51 @@
// Generates /llms.txt following https://llmstxt.org/ so that AI agents can
-// discover the documentation without scraping HTML.
+// discover the documentation without scraping HTML, plus a Markdown copy of
+// every indexed page next to its HTML (index.md beside index.html, as the spec
+// proposes) and /llms-full.txt, all of those pages in one file. llms-full.txt
+// is a widespread convention rather than part of the spec.
+// It was cheap to include here.
//
// For every component, only the version that 'stable' points to is indexed
// (the latest release, or nightly when building without release branches).
-// Each entry carries the published page URL, the page description and a link
+// Each entry has the Markdown page URL, the page description and a link
// to the AsciiDoc source on GitHub, so agents can both read a page and find
// the file to change.
//
+// A page with the attribute `:page-llms-ignore:` is left out of all of it.
+//
// Useful links:
// Extensions: https://docs.antora.org/antora/latest/extend/extensions/
// Types of events: https://docs.antora.org/antora/latest/extend/generator-events-reference/
+const { createConverter } = require('./html-to-markdown')
// Pages on how Stackable builds SDP go under "Optional", the llms.txt convention
// for links an agent can skip when context is short (https://llmstxt.org/).
+// They are left out of llms-full.txt for the same reason.
const OPTIONAL_MODULES = new Set(['contributor', 'internal'])
+// This is the entry point to the entire plugin.
+// Antora works in "phases" and here we hook into them.
module.exports.register = function () {
+ const markdown = new Map()
+
+ // The first phase we hook into is "documentsConverted" where Asciidoctor has
+ // turned every page into HTML.
+ // Page contents are only the article HTML at this point. Later in the build
+ // Antora wraps each page in the site layout, header and navigation included.
+ // (That phase is called "pagesComposed")
+ // And because we don't need all that ceremony we convert every HTML
+ // page to Markdown right here
+ this.once('documentsConverted', ({ playbook, contentCatalog }) => {
+ const siteUrl = (playbook.site.url || '').replace(/\/$/, '')
+ const pages = indexedPages(contentCatalog)
+ const convert = createConverter(new Map(pages.map((page) => [`${siteUrl}${page.pub.url}`, markdownUrl(page, siteUrl)])))
+ for (const page of pages) {
+ markdown.set(page, convert(page.contents.toString(), `${siteUrl}${page.pub.url}`))
+ }
+ })
+
+ // This is the second phase we hook into. The site is finished at this point and
+ // Antora is about to write the actual files to disk and we add our own files.
this.once('beforePublish', ({ playbook, contentCatalog, siteCatalog }) => {
const siteUrl = (playbook.site.url || '').replace(/\/$/, '')
const lines = [
@@ -28,24 +58,30 @@ module.exports.register = function () {
'Machine-readable release, component and CRD data is available from the',
'Stackable Hub: https://hub.stackable.tech/llms.txt',
'',
- 'Every entry below links the published page and, in parentheses, its AsciiDoc',
+ 'Every entry below links the page as Markdown and, in parentheses, its AsciiDoc',
'source on GitHub. To suggest a change to a page, edit that source file.',
+ `All pages outside "Optional" are also available in one file: ${siteUrl}/llms-full.txt`,
]
const optional = []
+ const full = []
+ // Here we iterate over every page of every component and add the markdown versions to the
+ // "siteCatalog" using their markdown path. So that's where Antora is going to publish them.
+ // We separate here between "optional" modules and non-optional ones.
+ // Every page gets its Markdown file (optional and non-optional).
+ // Pages from optional modules are listed under "Optional" at the end of llms.txt and left
+ // out of llms-full.txt. All others are listed per module and included in llms-full.txt.
for (const component of contentCatalog.getComponents()) {
- const version = component.latest
- const pages = contentCatalog
- .findBy({ component: component.name, version: version.version, family: 'page' })
- .filter((page) => page.out)
- .sort((a, b) => a.pub.url.localeCompare(b.pub.url))
const byModule = new Map()
- for (const page of pages) {
+ for (const page of indexedPages(contentCatalog, component)) {
+ const body = markdown.get(page)
+ siteCatalog.addFile({ contents: Buffer.from(markdownPage(page, body), 'utf8'), out: { path: markdownPath(page) } })
const module = page.src.module
if (OPTIONAL_MODULES.has(module)) {
optional.push(page)
continue
}
+ full.push(markdownPage(page, body, `${siteUrl}${page.pub.url}`))
if (!byModule.has(module)) byModule.set(module, [])
byModule.get(module).push(page)
}
@@ -60,21 +96,85 @@ module.exports.register = function () {
for (const page of optional) lines.push(entryFor(page, siteUrl))
}
- siteCatalog.addFile({
- contents: Buffer.from(lines.join('\n') + '\n', 'utf8'),
- out: { path: 'llms.txt' },
- })
+ siteCatalog.addFile({ contents: Buffer.from(lines.join('\n') + '\n', 'utf8'), out: { path: 'llms.txt' } })
+ siteCatalog.addFile({ contents: Buffer.from(full.join('\n\n'), 'utf8'), out: { path: 'llms-full.txt' } })
})
}
-function entryFor (page, siteUrl) {
- const title = decodeEntities(
- (page.asciidoc && page.asciidoc.doctitle ? page.asciidoc.doctitle : page.src.stem).replace(/<[^>]+>/g, '')
+// Sorted by the site navigation, which is the reading order we consciously chose.
+// This means that the markdown pages will also appear in "our" order in llms.txt later.
+// Pages that are not in the navigation come after it, sorted by URL.
+// The navigation only exists from navigationBuilt (another phase) on,
+// so earlier callers (like our "documentsConverted" call above) get URL order,
+// but for that call it doesn't matter because it only needs a full table of all pages
+// to convert them to markdown.
+function indexedPages (contentCatalog, component) {
+ const components = component ? [component] : contentCatalog.getComponents()
+ return components.flatMap((it) => {
+ const rank = navigationRank(it.latest.navigation || [])
+ const position = (page) => (rank.has(page.pub.url) ? rank.get(page.pub.url) : Infinity)
+
+ // Here we find all pages, then remove all those with a page-llms-ignore attribute
+ // and compare the rest by their "position" (see above) or URL as fallback.
+ return contentCatalog
+ .findBy({ component: it.name, version: it.latest.version, family: 'page' })
+ .filter((page) => page.out && !('page-llms-ignore' in page.asciidoc.attributes))
+ .sort((a, b) => {
+ if (position(a) !== position(b)) return position(a) - position(b)
+ return a.pub.url.localeCompare(b.pub.url)
+ })
+ })
+}
+
+// This builds an "order" of all pages by walking the entire tree of our pages.
+// As if you opened every section/subsection and noted down every page
+// along the way.
+function navigationRank (navigation) {
+ const rank = new Map()
+ const visit = (items) => {
+ for (const item of items) {
+ const url = item.urlType === 'internal' && item.url.split('#')[0]
+ if (url && !rank.has(url)) rank.set(url, rank.size)
+ visit(item.items || [])
+ }
+ }
+ visit(navigation)
+ return rank
+}
+
+function markdownPath (page) {
+ return page.out.path.replace(/\.html$/, '.md')
+}
+
+function markdownUrl (page, siteUrl) {
+ return `${siteUrl}/${markdownPath(page)}`
+}
+
+function titleOf (page) {
+ return decodeEntities(
+ (page.asciidoc.doctitle ? page.asciidoc.doctitle : page.src.stem).replace(/<[^>]+>/g, '')
)
- const description = page.asciidoc && page.asciidoc.attributes['description']
+}
+
+function descriptionOf (page) {
+ const description = page.asciidoc.attributes['description']
+ return description ? decodeEntities(description.replace(/\s+/g, ' ').trim()) : undefined
+}
+
+function markdownPage (page, body, htmlUrl) {
+ const head = [`# ${titleOf(page)}`]
+ if (htmlUrl) head.push(`URL: ${htmlUrl}`)
+ const description = descriptionOf(page)
+ if (description) head.push(`> ${description}`)
+ return `${head.join('\n\n')}\n\n${body}\n`
+}
+
+function entryFor (page, siteUrl) {
+ const url = markdownUrl(page, siteUrl)
+ const description = descriptionOf(page)
const source = sourceUrl(page.src)
- let entry = `- [${title}](${siteUrl}${page.pub.url})`
- if (description) entry += `: ${decodeEntities(description.replace(/\s+/g, ' ').trim())}`
+ let entry = `- [${titleOf(page)}](${url})`
+ if (description) entry += `: ${description}`
if (source) entry += ` (source: ${source})`
return entry
}
diff --git a/package-lock.json b/package-lock.json
index e605f2feb..d4081e1ed 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -9,7 +9,10 @@
],
"dependencies": {
"@asciidoctor/tabs": "^1.0.0-beta.6",
- "@sntke/antora-mermaid-extension": "0.0.8"
+ "@joplin/turndown-plugin-gfm": "1.0.68",
+ "@mixmark-io/domino": "2.2.0",
+ "@sntke/antora-mermaid-extension": "0.0.8",
+ "turndown": "7.2.4"
},
"devDependencies": {
"@antora/cli": "^3.1.7",
@@ -880,6 +883,12 @@
"dev": true,
"license": "ISC"
},
+ "node_modules/@joplin/turndown-plugin-gfm": {
+ "version": "1.0.68",
+ "resolved": "https://registry.npmjs.org/@joplin/turndown-plugin-gfm/-/turndown-plugin-gfm-1.0.68.tgz",
+ "integrity": "sha512-m8DfAQNC/V7g0j5H6Jv60WhekBBjodD+zTPGrU+g2m+ux/z8ZX07KQIl6kaAmbKwfvsQhC04op52g63I78sVgg==",
+ "license": "MIT"
+ },
"node_modules/@keyv/serialize": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.1.1.tgz",
@@ -887,6 +896,12 @@
"dev": true,
"license": "MIT"
},
+ "node_modules/@mixmark-io/domino": {
+ "version": "2.2.0",
+ "resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz",
+ "integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==",
+ "license": "BSD-2-Clause"
+ },
"node_modules/@napi-rs/wasm-runtime": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.6.tgz",
@@ -7362,6 +7377,19 @@
"license": "0BSD",
"optional": true
},
+ "node_modules/turndown": {
+ "version": "7.2.4",
+ "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.4.tgz",
+ "integrity": "sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ==",
+ "license": "MIT",
+ "dependencies": {
+ "@mixmark-io/domino": "^2.2.0"
+ },
+ "engines": {
+ "node": ">=18",
+ "npm": ">=9"
+ }
+ },
"node_modules/type-check": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
diff --git a/package.json b/package.json
index 0d9a8ac22..da4ccdf04 100644
--- a/package.json
+++ b/package.json
@@ -6,7 +6,10 @@
},
"dependencies": {
"@asciidoctor/tabs": "^1.0.0-beta.6",
- "@sntke/antora-mermaid-extension": "0.0.8"
+ "@joplin/turndown-plugin-gfm": "1.0.68",
+ "@mixmark-io/domino": "2.2.0",
+ "@sntke/antora-mermaid-extension": "0.0.8",
+ "turndown": "7.2.4"
},
"workspaces": [
"ui"