Refactor site generation into a modular content pipeline
Deploy website / deploy (push) Successful in 24s
Deploy website / deploy (push) Successful in 24s
This commit is contained in:
paul committed
1 parent
0f69028d4a
commit
146d516f24
20 files changed
+826
-427
No files matched your search
+36
-339
@@ -1,68 +1,21 @@
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { mkdir, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises'
|
||||
import { mkdir, readFile, writeFile } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { escapeHtml, h, raw, render } from './html.js'
|
||||
import { marked } from 'marked'
|
||||
import { dateFor, readPages } from './content.js'
|
||||
import { renderFeed, renderSitemap, toSitemapDate } from './feed.js'
|
||||
import { createOutput, discardOutput, publishOutput } from './output.js'
|
||||
import { renderMarkdownPage } from './markdown.js'
|
||||
import {
|
||||
contentDirectories,
|
||||
generatedRoot,
|
||||
navLabels,
|
||||
siteName,
|
||||
siteUrl,
|
||||
standalonePages,
|
||||
} from './site.config.js'
|
||||
|
||||
const siteName = 'PaulW.XYZ'
|
||||
const siteUrl = 'https://paulw.xyz'
|
||||
const output = '.generated'
|
||||
const markdownDirs = ['notes', 'posts']
|
||||
const includeDrafts = process.argv.includes('--drafts')
|
||||
const navLabels = {
|
||||
notes: 'Notes',
|
||||
posts: 'Posts',
|
||||
about: 'About',
|
||||
licenses: 'Licenses',
|
||||
}
|
||||
|
||||
const standalonePages = [
|
||||
{ source: 'README.md', path: 'about', title: 'About' },
|
||||
{ source: 'THIRD_PARTY_LICENSES.md', path: 'licenses', title: 'Licenses' },
|
||||
]
|
||||
|
||||
const languageLabels = {
|
||||
ini: 'INI',
|
||||
js: 'JavaScript',
|
||||
javascript: 'JavaScript',
|
||||
lua: 'Lua',
|
||||
sh: 'Shell',
|
||||
bash: 'Shell',
|
||||
}
|
||||
|
||||
const slug = (value) => {
|
||||
let result = ''
|
||||
let dash = false
|
||||
for (const character of String(value).toLowerCase()) {
|
||||
const valid = (character >= 'a' && character <= 'z') || (character >= '0' && character <= '9')
|
||||
if (valid) {
|
||||
if (dash && result) result += '-'
|
||||
result += character
|
||||
dash = false
|
||||
} else if (result) dash = true
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
const headingLabel = (tokens) =>
|
||||
tokens
|
||||
.map((token) => {
|
||||
if (token.type === 'image') return token.text ?? ''
|
||||
if (token.tokens) return headingLabel(token.tokens)
|
||||
return token.text ?? token.raw ?? ''
|
||||
})
|
||||
.join('')
|
||||
.trim()
|
||||
|
||||
const createSlugger = () => {
|
||||
const counts = new Map()
|
||||
return (tokens) => {
|
||||
const base = slug(headingLabel(tokens)) || 'section'
|
||||
const count = counts.get(base) ?? 0
|
||||
counts.set(base, count + 1)
|
||||
return count === 0 ? base : `${base}-${count + 1}`
|
||||
}
|
||||
}
|
||||
const output = await createOutput(generatedRoot)
|
||||
|
||||
const pageShell = (title, body, path = '/', navTitle = '', { draft = false } = {}) =>
|
||||
`<!DOCTYPE html>${render(
|
||||
@@ -106,272 +59,6 @@ const nav = (path, currentTitle = '') => {
|
||||
return render(children)
|
||||
}
|
||||
|
||||
const dateFor = async (file) => {
|
||||
try {
|
||||
const gitDate = execFileSync(
|
||||
'git',
|
||||
['log', '-1', '--date=format-local:%Y-%m-%dT%H:%M:%SZ', '--format=%ad', '--', file],
|
||||
{ env: { ...process.env, TZ: 'UTC' } },
|
||||
)
|
||||
.toString()
|
||||
.trim()
|
||||
if (gitDate) return gitDate
|
||||
} catch {}
|
||||
try {
|
||||
return (await stat(file)).mtime.toISOString()
|
||||
} catch {
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
const readPages = async (dir) => {
|
||||
let entries
|
||||
try {
|
||||
entries = await readdir(dir, { withFileTypes: true })
|
||||
} catch (error) {
|
||||
if (error.code === 'ENOENT') return []
|
||||
throw error
|
||||
}
|
||||
const markdownEntries = entries
|
||||
.filter((entry) => {
|
||||
if (!entry.isFile() || !entry.name.endsWith('.md') || entry.name === '.md') return false
|
||||
return includeDrafts || !entry.name.startsWith('.')
|
||||
})
|
||||
.sort((a, b) => a.name.localeCompare(b.name))
|
||||
const pages = await Promise.all(
|
||||
markdownEntries.map(async (entry) => {
|
||||
const file = join(dir, entry.name)
|
||||
const source = await readFile(file, 'utf8')
|
||||
const titleLine = source.split(/\r?\n/).find((line) => line.startsWith('# '))
|
||||
if (!titleLine) return undefined
|
||||
const draft = entry.name.startsWith('.')
|
||||
const name = (draft ? entry.name.slice(1) : entry.name).slice(0, -3)
|
||||
if (!name) return undefined
|
||||
return {
|
||||
name,
|
||||
title: stripSidenotes(titleLine.slice(2).trim()),
|
||||
file,
|
||||
source,
|
||||
date: await dateFor(file),
|
||||
draft,
|
||||
}
|
||||
}),
|
||||
)
|
||||
const byName = new Map()
|
||||
for (const page of pages.filter(Boolean)) {
|
||||
const existing = byName.get(page.name)
|
||||
if (!existing || (existing.draft && !page.draft)) byName.set(page.name, page)
|
||||
}
|
||||
return [...byName.values()].sort((a, b) => a.name.localeCompare(b.name))
|
||||
}
|
||||
|
||||
let nextHeadingId = createSlugger()
|
||||
let sidenoteNumber = 0
|
||||
|
||||
marked.use({
|
||||
extensions: [
|
||||
{
|
||||
name: 'sidenote',
|
||||
level: 'inline',
|
||||
start: (source) => source.indexOf('^['),
|
||||
tokenizer(source) {
|
||||
if (!source.startsWith('^[')) return undefined
|
||||
let depth = 1
|
||||
for (let index = 2; index < source.length; index += 1) {
|
||||
if (source[index] === '\\') {
|
||||
index += 1
|
||||
} else if (source[index] === '[') {
|
||||
depth += 1
|
||||
} else if (source[index] === ']') {
|
||||
depth -= 1
|
||||
if (depth === 0) {
|
||||
const text = source.slice(2, index)
|
||||
return {
|
||||
type: 'sidenote',
|
||||
raw: source.slice(0, index + 1),
|
||||
tokens: this.lexer.inlineTokens(text),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined
|
||||
},
|
||||
renderer({ tokens }) {
|
||||
sidenoteNumber += 1
|
||||
const id = `sidenote-${sidenoteNumber}`
|
||||
const content = this.parser.parseInline(tokens)
|
||||
return `<input type="checkbox" id="${id}" class="sidenote-toggle visually-hidden"><label for="${id}" class="margin-toggle sidenote-number"><span class="visually-hidden">Toggle sidenote ${sidenoteNumber}</span></label><span class="sidenote">${content}</span>`
|
||||
},
|
||||
},
|
||||
],
|
||||
renderer: {
|
||||
code({ text, lang, escaped }) {
|
||||
const language = lang?.trim().split(/\s+/, 1)[0].toLowerCase() || 'plain'
|
||||
const label = languageLabels[language] ?? language.charAt(0).toUpperCase() + language.slice(1)
|
||||
const source = escaped ? text : escapeHtml(text.replace(/\n$/, ''))
|
||||
return `<div class="code-block"><div class="code-block-header"><span>${escapeHtml(label)}</span><button type="button" class="copy-code" aria-label="Copy ${escapeHtml(label)} code" aria-live="polite">Copy</button></div><pre><code class="language-${escapeHtml(language)}">${source}\n</code></pre></div>\n`
|
||||
},
|
||||
heading({ tokens, depth }) {
|
||||
const text = this.parser.parseInline(tokens)
|
||||
const id = nextHeadingId(tokens)
|
||||
return `<h${depth} id="${id}">${text}</h${depth}>\n`
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const withoutTitle = (source) => {
|
||||
const lines = source.split('\n')
|
||||
const index = lines.findIndex((line) => line.startsWith('# '))
|
||||
if (index >= 0) lines.splice(index, 1)
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
const stripSidenotes = (text) => {
|
||||
let result = ''
|
||||
for (let index = 0; index < text.length;) {
|
||||
if (text[index] === '^' && text[index + 1] === '[') {
|
||||
const end = text.indexOf(']', index + 2)
|
||||
if (end >= 0) {
|
||||
index = end + 1
|
||||
continue
|
||||
}
|
||||
}
|
||||
result += text[index++]
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
const stripHeadingSidenotes = (source) =>
|
||||
source
|
||||
.split('\n')
|
||||
.map((line) => {
|
||||
const content = line.trimStart()
|
||||
return content.startsWith('# ') ||
|
||||
content.startsWith('## ') ||
|
||||
content.startsWith('### ') ||
|
||||
content.startsWith('#### ') ||
|
||||
content.startsWith('##### ') ||
|
||||
content.startsWith('###### ')
|
||||
? line.slice(0, line.length - content.length) + stripSidenotes(content)
|
||||
: line
|
||||
})
|
||||
.join('\n')
|
||||
|
||||
const renderMarkdown = (source) => {
|
||||
nextHeadingId = createSlugger()
|
||||
sidenoteNumber = 0
|
||||
const html = marked.parse(withoutTitle(stripHeadingSidenotes(source)))
|
||||
let result = ''
|
||||
let cursor = 0
|
||||
while (cursor < html.length) {
|
||||
const start = html.indexOf('<table', cursor)
|
||||
if (start < 0) {
|
||||
result += html.slice(cursor)
|
||||
break
|
||||
}
|
||||
const end = html.indexOf('</table>', start)
|
||||
if (end < 0) {
|
||||
result += html.slice(cursor)
|
||||
break
|
||||
}
|
||||
result += html.slice(cursor, start)
|
||||
result += `<div class="table-scroll">${html.slice(start, end + 8)}</div>`
|
||||
cursor = end + 8
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
const toc = (source) => {
|
||||
const headings = marked.lexer(withoutTitle(stripHeadingSidenotes(source))).filter((token) => token.type === 'heading')
|
||||
if (headings.length === 0) return ''
|
||||
const root = { depth: 0, children: [] }
|
||||
const stack = [root]
|
||||
const headingId = createSlugger()
|
||||
for (const heading of headings) {
|
||||
while (stack.length > 1 && stack.at(-1).depth >= heading.depth) stack.pop()
|
||||
const node = { depth: heading.depth, heading, children: [] }
|
||||
stack.at(-1).children.push(node)
|
||||
stack.push(node)
|
||||
}
|
||||
const renderItems = (items) =>
|
||||
`<ol>${items
|
||||
.map(({ heading, children }) => {
|
||||
const id = headingId(heading.tokens)
|
||||
const nested = children.length > 0 ? renderItems(children) : ''
|
||||
return `<li><a href="#${id}">${escapeHtml(headingLabel(heading.tokens))}</a>${nested}</li>`
|
||||
})
|
||||
.join('')}</ol>`
|
||||
return renderItems(root.children)
|
||||
}
|
||||
|
||||
const excerpt = (source, max = 280) => {
|
||||
const paragraph = marked
|
||||
.lexer(withoutTitle(stripHeadingSidenotes(source)))
|
||||
.find((token) => token.type === 'paragraph')
|
||||
if (!paragraph) return ''
|
||||
const text = stripSidenotes(headingLabel(paragraph.tokens ?? [{ text: paragraph.text ?? '' }]))
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
if (text.length <= max) return text
|
||||
const slice = text.slice(0, max)
|
||||
const bound = slice.lastIndexOf(' ')
|
||||
return `${(bound > 80 ? slice.slice(0, bound) : slice).trimEnd()}…`
|
||||
}
|
||||
|
||||
const toRssDate = (iso) => {
|
||||
if (!iso) return ''
|
||||
const date = new Date(iso)
|
||||
return Number.isNaN(date.valueOf()) ? '' : date.toUTCString()
|
||||
}
|
||||
|
||||
const toSitemapDate = (iso) => (iso ? iso.slice(0, 10) : '')
|
||||
|
||||
const renderFeed = (items) => {
|
||||
const lastBuild = toRssDate(items.find((item) => item.date)?.date) || new Date().toUTCString()
|
||||
const itemXml = items
|
||||
.map((item) => {
|
||||
const loc = `${siteUrl}/${item.dir}/${item.name}/`
|
||||
const description = excerpt(item.source)
|
||||
const pubDate = toRssDate(item.date)
|
||||
return `<item>
|
||||
<title>${escapeHtml(item.title)}</title>
|
||||
<link>${escapeHtml(loc)}</link>
|
||||
<guid isPermaLink="true">${escapeHtml(loc)}</guid>
|
||||
${pubDate ? `<pubDate>${escapeHtml(pubDate)}</pubDate>` : ''}
|
||||
<category>${escapeHtml(item.dir)}</category>
|
||||
${description ? `<description>${escapeHtml(description)}</description>` : ''}
|
||||
</item>`
|
||||
})
|
||||
.join('\n ')
|
||||
|
||||
return `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
|
||||
<channel>
|
||||
<title>${escapeHtml(siteName)}</title>
|
||||
<link>${escapeHtml(`${siteUrl}/`)}</link>
|
||||
<description>${escapeHtml(`Notes and posts from ${siteName}`)}</description>
|
||||
<language>en-US</language>
|
||||
<atom:link href="${escapeHtml(`${siteUrl}/feed.xml`)}" rel="self" type="application/rss+xml"/>
|
||||
<lastBuildDate>${escapeHtml(lastBuild)}</lastBuildDate>
|
||||
${itemXml}
|
||||
</channel>
|
||||
</rss>
|
||||
`
|
||||
}
|
||||
|
||||
const renderSitemap = (urls) => `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
||||
${urls
|
||||
.map(({ loc, lastmod }) => {
|
||||
const modified = lastmod ? `\n <lastmod>${escapeHtml(lastmod)}</lastmod>` : ''
|
||||
return ` <url>
|
||||
<loc>${escapeHtml(loc)}</loc>${modified}
|
||||
</url>`
|
||||
})
|
||||
.join('\n')}
|
||||
</urlset>
|
||||
`
|
||||
|
||||
const writePage = async (path, html) => {
|
||||
const directory = join(output, path)
|
||||
await mkdir(directory, { recursive: true })
|
||||
@@ -407,7 +94,7 @@ const renderNotesTable = (pages, directory) => {
|
||||
</tr>`,
|
||||
)
|
||||
.join('')
|
||||
return `<table class="${directory}-table"><caption class="visually-hidden">${escapeHtml(directory)} pages</caption><tbody>${rows}</tbody></table>`
|
||||
return `<table class="${directory}-table"><caption class="visually-hidden">${escapeHtml(directory)} pages</caption><thead><tr><th scope="col">Page</th><th scope="col">Last updated</th></tr></thead><tbody>${rows}</tbody></table>`
|
||||
}
|
||||
|
||||
const renderPostsList = (pages, directory) => {
|
||||
@@ -425,7 +112,7 @@ const renderPostsList = (pages, directory) => {
|
||||
|
||||
const renderEmptySection = () => `
|
||||
<div class="text center">
|
||||
<p>**crickets**</p>
|
||||
<p><strong>crickets</strong></p>
|
||||
<p>Nothing to list...</p>
|
||||
<a class="link button back" href="/">Go Home</a>
|
||||
</div>`
|
||||
@@ -463,19 +150,21 @@ const renderArticle = (directory, page, navigationPath = `/${directory}/${page.n
|
||||
const draftNotice = page.draft
|
||||
? `<p class="draft-banner" role="status">Draft — rename <code>.${escapeHtml(page.name)}.md</code> to <code>${escapeHtml(page.name)}.md</code> to publish.</p>`
|
||||
: ''
|
||||
const tableOfContents = toc(page.source)
|
||||
const tableOfContents = page.markdown.toc
|
||||
const contents = tableOfContents
|
||||
? `<section class="block toc"><details open><summary>Contents</summary>${tableOfContents}</details></section><hr>`
|
||||
: ''
|
||||
const article = `<section class="block">${renderMarkdown(page.source)}</section>`
|
||||
const article = `<section class="block">${page.markdown.html}</section>`
|
||||
const body = `${heading}<main class="article-page ${directory}-page container">${draftNotice}${updated}${contents}${article}</main>`
|
||||
|
||||
return pageShell(page.title, body, navigationPath, page.title, { draft: Boolean(page.draft) })
|
||||
}
|
||||
|
||||
await rm(output, { recursive: true, force: true })
|
||||
await mkdir(output, { recursive: true })
|
||||
const sections = Object.fromEntries(await Promise.all(markdownDirs.map(async (dir) => [dir, await readPages(dir)])))
|
||||
try {
|
||||
const sections = Object.fromEntries(await Promise.all(contentDirectories.map(async (dir) => [dir, await readPages(dir, { includeDrafts })])))
|
||||
for (const pages of Object.values(sections)) {
|
||||
for (const page of pages) page.markdown = renderMarkdownPage(page.source)
|
||||
}
|
||||
|
||||
const external = JSON.parse(await readFile('external.json', 'utf8'))
|
||||
|
||||
@@ -483,7 +172,7 @@ const externalLinks = Object.entries(external)
|
||||
.map(([text, href]) => `<a class="button blue link extern" href="${escapeHtml(href)}">${escapeHtml(text)}</a>`)
|
||||
.join('')
|
||||
|
||||
const sectionLinks = markdownDirs
|
||||
const sectionLinks = contentDirectories
|
||||
.map((dir) => `<a class="button link" href="/${dir}/">${dir[0].toUpperCase() + dir.slice(1)}</a>`)
|
||||
.join('')
|
||||
|
||||
@@ -492,7 +181,7 @@ const standaloneLinks = standalonePages
|
||||
.map((page) => `<a class="button link" href="/${page.path}/">${page.title}</a>`)
|
||||
.join('')
|
||||
|
||||
const recentPages = markdownDirs
|
||||
const recentPages = contentDirectories
|
||||
.flatMap((dir) => sections[dir].filter((page) => !page.draft).map((page) => ({ ...page, dir })))
|
||||
.sort((a, b) => b.date.localeCompare(a.date))
|
||||
.slice(0, 5)
|
||||
@@ -506,7 +195,7 @@ const recentLinks = recentPages
|
||||
await writeFile(join(output, 'index.html'), renderHome(externalLinks, sectionLinks + standaloneLinks, recentLinks))
|
||||
await writeFile(join(output, '404.html'), renderNotFound())
|
||||
|
||||
for (const dir of markdownDirs) {
|
||||
for (const dir of contentDirectories) {
|
||||
const pages = sections[dir]
|
||||
await writePage(
|
||||
dir,
|
||||
@@ -528,16 +217,19 @@ for (const page of standalonePages) {
|
||||
name: page.path,
|
||||
title: page.title,
|
||||
source,
|
||||
markdown: renderMarkdownPage(source),
|
||||
date: await dateFor(page.source),
|
||||
}
|
||||
generatedStandalone.push(generatedPage)
|
||||
await writePage(page.path, renderArticle(page.path, generatedPage, '/'))
|
||||
}
|
||||
|
||||
const feedItems = markdownDirs
|
||||
const feedItems = contentDirectories
|
||||
.flatMap((dir) => sections[dir].filter((page) => !page.draft).map((page) => ({ ...page, dir })))
|
||||
.sort((a, b) => b.date.localeCompare(a.date))
|
||||
|
||||
for (const item of feedItems) item.description = item.markdown.excerpt
|
||||
|
||||
await writeFile(join(output, 'feed.xml'), renderFeed(feedItems))
|
||||
|
||||
const sitemapUrls = [
|
||||
@@ -545,7 +237,7 @@ const sitemapUrls = [
|
||||
loc: `${siteUrl}/`,
|
||||
lastmod: toSitemapDate(feedItems[0]?.date),
|
||||
},
|
||||
...markdownDirs.map((dir) => ({
|
||||
...contentDirectories.map((dir) => ({
|
||||
loc: `${siteUrl}/${dir}/`,
|
||||
lastmod: toSitemapDate(
|
||||
[...sections[dir]].filter((page) => !page.draft).sort((a, b) => b.date.localeCompare(a.date))[0]?.date,
|
||||
@@ -563,3 +255,8 @@ const sitemapUrls = [
|
||||
]
|
||||
|
||||
await writeFile(join(output, 'sitemap.xml'), renderSitemap(sitemapUrls))
|
||||
await publishOutput(output, generatedRoot)
|
||||
} catch (error) {
|
||||
await discardOutput(output)
|
||||
throw error
|
||||
}
|
||||
Reference in new issue
Block a user