Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
155 changes: 155 additions & 0 deletions lib/html-to-markdown.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
// Converts the HTML Asciidoctor renders for a page body into Markdown.
//
// Asciidoctor lays out admonitions and callout lists as tables, so without the
// rules below they would come out as Markdown tables with an empty icon column.

// Domino is a browser DOM in Node. So it can parse HTML into a tree of elements.
const domino = require('@mixmark-io/domino')
// Turndown converts a DOM tree of elements into Markdown
const TurndownService = require('turndown')
// Lets turndown write HTML tables as GitHub-style Markdown tables
const { tables } = require('@joplin/turndown-plugin-gfm')

const ADMONITION_LABELS = {
note: 'Note',
tip: 'Tip',
important: 'Important',
warning: 'Warning',
caution: 'Caution',
}

function hasClass (node, name) {
return node.nodeType === 1 && (node.getAttribute('class') || '').split(/\s+/).includes(name)
}

// domino, the DOM turndown runs on in Node, returns lists that are not iterable.
function elements (node, tag) {
return Array.from(node.getElementsByTagName(tag))
}

function quote (text) {
return text
.trim()
.split('\n')
.map((line) => (line ? `> ${line}` : '>'))
.join('\n')
}

function fence (text, info = '') {
const longestRun = Math.max(0, ...(text.match(/`+/g) || []).map((run) => run.length))
const marker = '`'.repeat(Math.max(3, longestRun + 1))
return `\n\n${marker}${info}\n${text.replace(/\n$/, '')}\n${marker}\n\n`
}

// Rewrites the DOM before turndown sees it. Links and images become absolute,
// because llms-full.txt has no directory to resolve relative ones from.
// Links to an indexed page go to its Markdown version, without the fragment,
// because the Markdown has no anchor IDs.
// The Mermaid source becomes a pre, because that is the only element where
// turndown keeps whitespace. The <br>s in its node labels are parsed as
// elements, so they are turned back into text.
function prepare (html, pageUrl, markdownUrls) {
const document = domino.createDocument(`<!doctype html><html><body>${html}</body></html>`)
const base = /^https?:/.test(pageUrl || '') ? pageUrl : undefined
const absolute = (url) => (!base || /^[a-z][a-z0-9+.-]*:/i.test(url) ? url : new URL(url, base).href)
for (const [tag, attribute] of [['a', 'href'], ['img', 'src']]) {
for (const node of elements(document.body, tag)) {
const value = node.getAttribute(attribute)
if (value) node.setAttribute(attribute, absolute(value))
}
}
for (const node of elements(document.body, 'a')) {
const markdownUrl = markdownUrls.get((node.getAttribute('href') || '').split('#')[0])
if (markdownUrl) node.setAttribute('href', markdownUrl)
}
for (const node of elements(document.body, 'div').filter((div) => hasClass(div, 'mermaid'))) {
for (const br of elements(node, 'br')) br.parentNode.replaceChild(document.createTextNode('<br>'), br)
const pre = document.createElement('pre')
pre.setAttribute('class', 'mermaid')
pre.textContent = node.textContent
node.parentNode.replaceChild(pre, node)
}
return document.body
}

// This is the entry point into the file.
// It creates a TurndownService which walks a DOM element tree and writes Markdown.
// `<h2>` becomes `##` and so on.
// We have to add a bunch of rules and plugins which know the details of the
// Asciidoctor output and the HTML it creates.
function createConverter (markdownUrls = new Map()) {
const turndown = new TurndownService({
headingStyle: 'atx',
codeBlockStyle: 'fenced',
bulletListMarker: '-',
emDelimiter: '_',
})
turndown.use(tables)
turndown.remove(['style', 'script'])

// Section anchors are empty links that only show a CSS glyph. This needs a rule
// instead of remove(), because turndown checks remove() after its link rule.
turndown.addRule('sectionAnchor', {
filter: (node) => node.nodeName === 'A' && hasClass(node, 'anchor'),
replacement: () => '',
})

// Asciidoctor puts the language in data-lang. The class that turndown reads by
// default says "language-none" on untagged blocks. Literal blocks are a bare pre.
turndown.addRule('codeBlock', {
filter: (node) => node.nodeName === 'PRE',
replacement: (_content, node) => {
if (hasClass(node, 'mermaid')) return fence(node.textContent.trim(), 'mermaid')
const code = node.firstChild && node.firstChild.nodeName === 'CODE' ? node.firstChild : node
return fence(code.textContent, code.getAttribute('data-lang') || '')
},
})

// Asciidoctor renders a tab set as a list of labels followed by the panels.
// Without its label in front of each panel, the alternatives read like steps.
turndown.addRule('tabs', {
filter: (node) => node.nodeName === 'DIV' && hasClass(node, 'tabs'),
replacement: (_content, node) => {
const labels = new Map(elements(node, 'li').map((li) => [li.getAttribute('id'), li.textContent.trim()]))
const panels = elements(node, 'div').filter(
(div) => hasClass(div, 'tabpanel') && div.parentNode.parentNode === node
)
const sections = panels.map((panel) => {
const label = labels.get(panel.getAttribute('aria-labelledby')) || ''
return `**Tab: ${label}**\n\n${turndown.turndown(panel.innerHTML)}`
})
return `\n\n${sections.join('\n\n')}\n\n`
},
})

turndown.addRule('admonition', {
filter: (node) => node.nodeName === 'DIV' && hasClass(node, 'admonitionblock'),
replacement: (_content, node) => {
const type = Object.keys(ADMONITION_LABELS).find((name) => hasClass(node, name))
const cell = elements(node, 'td').find((td) => hasClass(td, 'content'))
const body = turndown.turndown(cell.innerHTML)
return `\n\n${quote(`**${ADMONITION_LABELS[type]}:** ${body}`)}\n\n`
},
})

turndown.addRule('calloutList', {
filter: (node) => node.nodeName === 'DIV' && hasClass(node, 'colist'),
replacement: (_content, node) => {
const rows = elements(node, 'tr').map((row) => {
const [marker, text] = elements(row, 'td')
return `(${marker.textContent.trim()}) ${turndown.turndown(text.innerHTML).replace(/\n+/g, ' ')}`
})
return `\n\n${rows.join('\n')}\n\n`
},
})

turndown.addRule('blockTitle', {
filter: (node) => node.nodeName === 'DIV' && hasClass(node, 'title'),
replacement: (content) => `\n\n**${content.trim()}**\n\n`,
})

// Paragraphs inside list items leave whitespace-only lines behind.
return (html, pageUrl) => turndown.turndown(prepare(html, pageUrl, markdownUrls)).replace(/[ \t]+$/gm, '')
}

module.exports = { createConverter }
138 changes: 119 additions & 19 deletions lib/llms-txt.js
Original file line number Diff line number Diff line change
@@ -1,21 +1,51 @@
// Generates /llms.txt following https://llmstxt.org/ so that AI agents can
// discover the documentation without scraping HTML.
// discover the documentation without scraping HTML, plus a Markdown copy of
// every indexed page next to its HTML (index.md beside index.html, as the spec
// proposes) and /llms-full.txt, all of those pages in one file. llms-full.txt
// is a widespread convention rather than part of the spec.
// It was cheap to include here.
//
// For every component, only the version that 'stable' points to is indexed
// (the latest release, or nightly when building without release branches).
// Each entry carries the published page URL, the page description and a link
// Each entry has the Markdown page URL, the page description and a link
// to the AsciiDoc source on GitHub, so agents can both read a page and find
// the file to change.
//
// A page with the attribute `:page-llms-ignore:` is left out of all of it.
//
// Useful links:
// Extensions: https://docs.antora.org/antora/latest/extend/extensions/
// Types of events: https://docs.antora.org/antora/latest/extend/generator-events-reference/
const { createConverter } = require('./html-to-markdown')

// Pages on how Stackable builds SDP go under "Optional", the llms.txt convention
// for links an agent can skip when context is short (https://llmstxt.org/).
// They are left out of llms-full.txt for the same reason.
const OPTIONAL_MODULES = new Set(['contributor', 'internal'])

// This is the entry point to the entire plugin.
// Antora works in "phases" and here we hook into them.
module.exports.register = function () {
const markdown = new Map()

// The first phase we hook into is "documentsConverted" where Asciidoctor has
// turned every page into HTML.
// Page contents are only the article HTML at this point. Later in the build
// Antora wraps each page in the site layout, header and navigation included.
// (That phase is called "pagesComposed")
// And because we don't need all that ceremony we convert every HTML
// page to Markdown right here
this.once('documentsConverted', ({ playbook, contentCatalog }) => {
const siteUrl = (playbook.site.url || '').replace(/\/$/, '')
const pages = indexedPages(contentCatalog)
const convert = createConverter(new Map(pages.map((page) => [`${siteUrl}${page.pub.url}`, markdownUrl(page, siteUrl)])))
for (const page of pages) {
markdown.set(page, convert(page.contents.toString(), `${siteUrl}${page.pub.url}`))
}
})

// This is the second phase we hook into. The site is finished at this point and
// Antora is about to write the actual files to disk and we add our own files.
this.once('beforePublish', ({ playbook, contentCatalog, siteCatalog }) => {
const siteUrl = (playbook.site.url || '').replace(/\/$/, '')
const lines = [
Expand All @@ -28,24 +58,30 @@ module.exports.register = function () {
'Machine-readable release, component and CRD data is available from the',
'Stackable Hub: https://hub.stackable.tech/llms.txt',
'',
'Every entry below links the published page and, in parentheses, its AsciiDoc',
'Every entry below links the page as Markdown and, in parentheses, its AsciiDoc',
'source on GitHub. To suggest a change to a page, edit that source file.',
`All pages outside "Optional" are also available in one file: ${siteUrl}/llms-full.txt`,
]
const optional = []
const full = []

// Here we iterate over every page of every component and add the markdown versions to the
// "siteCatalog" using their markdown path. So that's where Antora is going to publish them.
// We separate here between "optional" modules and non-optional ones.
// Every page gets its Markdown file (optional and non-optional).
// Pages from optional modules are listed under "Optional" at the end of llms.txt and left
// out of llms-full.txt. All others are listed per module and included in llms-full.txt.
for (const component of contentCatalog.getComponents()) {
const version = component.latest
const pages = contentCatalog
.findBy({ component: component.name, version: version.version, family: 'page' })
.filter((page) => page.out)
.sort((a, b) => a.pub.url.localeCompare(b.pub.url))
const byModule = new Map()
for (const page of pages) {
for (const page of indexedPages(contentCatalog, component)) {
const body = markdown.get(page)
siteCatalog.addFile({ contents: Buffer.from(markdownPage(page, body), 'utf8'), out: { path: markdownPath(page) } })
const module = page.src.module
if (OPTIONAL_MODULES.has(module)) {
optional.push(page)
continue
}
full.push(markdownPage(page, body, `${siteUrl}${page.pub.url}`))
if (!byModule.has(module)) byModule.set(module, [])
byModule.get(module).push(page)
}
Expand All @@ -60,21 +96,85 @@ module.exports.register = function () {
for (const page of optional) lines.push(entryFor(page, siteUrl))
}

siteCatalog.addFile({
contents: Buffer.from(lines.join('\n') + '\n', 'utf8'),
out: { path: 'llms.txt' },
})
siteCatalog.addFile({ contents: Buffer.from(lines.join('\n') + '\n', 'utf8'), out: { path: 'llms.txt' } })
siteCatalog.addFile({ contents: Buffer.from(full.join('\n\n'), 'utf8'), out: { path: 'llms-full.txt' } })
})
}

function entryFor (page, siteUrl) {
const title = decodeEntities(
(page.asciidoc && page.asciidoc.doctitle ? page.asciidoc.doctitle : page.src.stem).replace(/<[^>]+>/g, '')
// Sorted by the site navigation, which is the reading order we consciously chose.
// This means that the markdown pages will also appear in "our" order in llms.txt later.
// Pages that are not in the navigation come after it, sorted by URL.
// The navigation only exists from navigationBuilt (another phase) on,
// so earlier callers (like our "documentsConverted" call above) get URL order,
// but for that call it doesn't matter because it only needs a full table of all pages
// to convert them to markdown.
function indexedPages (contentCatalog, component) {
const components = component ? [component] : contentCatalog.getComponents()
return components.flatMap((it) => {
const rank = navigationRank(it.latest.navigation || [])
const position = (page) => (rank.has(page.pub.url) ? rank.get(page.pub.url) : Infinity)

// Here we find all pages, then remove all those with a page-llms-ignore attribute
// and compare the rest by their "position" (see above) or URL as fallback.
return contentCatalog
.findBy({ component: it.name, version: it.latest.version, family: 'page' })
.filter((page) => page.out && !('page-llms-ignore' in page.asciidoc.attributes))
.sort((a, b) => {
if (position(a) !== position(b)) return position(a) - position(b)
return a.pub.url.localeCompare(b.pub.url)
})
})
}

// This builds an "order" of all pages by walking the entire tree of our pages.
// As if you opened every section/subsection and noted down every page
// along the way.
function navigationRank (navigation) {
const rank = new Map()
const visit = (items) => {
for (const item of items) {
const url = item.urlType === 'internal' && item.url.split('#')[0]
if (url && !rank.has(url)) rank.set(url, rank.size)
visit(item.items || [])
}
}
visit(navigation)
return rank
}

function markdownPath (page) {
return page.out.path.replace(/\.html$/, '.md')
}

function markdownUrl (page, siteUrl) {
return `${siteUrl}/${markdownPath(page)}`
}

function titleOf (page) {
return decodeEntities(
(page.asciidoc.doctitle ? page.asciidoc.doctitle : page.src.stem).replace(/<[^>]+>/g, '')
)
const description = page.asciidoc && page.asciidoc.attributes['description']
}

function descriptionOf (page) {
const description = page.asciidoc.attributes['description']
return description ? decodeEntities(description.replace(/\s+/g, ' ').trim()) : undefined
}

function markdownPage (page, body, htmlUrl) {
const head = [`# ${titleOf(page)}`]
if (htmlUrl) head.push(`URL: ${htmlUrl}`)
const description = descriptionOf(page)
if (description) head.push(`> ${description}`)
return `${head.join('\n\n')}\n\n${body}\n`
}

function entryFor (page, siteUrl) {
const url = markdownUrl(page, siteUrl)
const description = descriptionOf(page)
const source = sourceUrl(page.src)
let entry = `- [${title}](${siteUrl}${page.pub.url})`
if (description) entry += `: ${decodeEntities(description.replace(/\s+/g, ' ').trim())}`
let entry = `- [${titleOf(page)}](${url})`
if (description) entry += `: ${description}`
if (source) entry += ` (source: ${source})`
return entry
}
Expand Down
Loading