From 0f69028d4ac25770511f1fdc398289070adc45e7 Mon Sep 17 00:00:00 2001 From: "Paul W." Date: Fri, 4 Sep 2026 21:41:52 -0400 Subject: [PATCH] Add RSS, sitemap, and local dotted-file drafts. --- check.js | 61 +++++++++++++++- generator.js | 173 ++++++++++++++++++++++++++++++++++++++++++---- package.json | 2 +- public/robots.txt | 4 ++ src/global.css | 11 +++ vite.config.js | 19 ++++- 6 files changed, 252 insertions(+), 18 deletions(-) create mode 100644 public/robots.txt diff --git a/check.js b/check.js index 30b9ea7..fc3e1f7 100644 --- a/check.js +++ b/check.js @@ -1,3 +1,4 @@ +import { execFileSync } from 'node:child_process' import { readdir, readFile, stat } from 'node:fs/promises' import { join } from 'node:path' @@ -24,6 +25,7 @@ const exists = async (path) => { const errors = [] const htmlFiles = (await walk(generatedRoot)).filter((file) => file.endsWith('.html')) +const noindexFiles = new Set() const clientSource = await readFile('src/main.js', 'utf8') const builtInPrismLanguages = new Set([ 'markup', @@ -48,6 +50,10 @@ for (const file of htmlFiles) { if (duplicateIds.length > 0) errors.push(`${file}: duplicate IDs: ${[...new Set(duplicateIds)].join(', ')}`) if (!html.includes(' element`) + if (html.includes('name="robots"') && html.includes('noindex')) { + noindexFiles.add(file) + errors.push(`${file}: draft page must not be in production output`) + } for (const [, fragment] of html.matchAll(/href="#([^"]+)"/g)) { if (!ids.includes(fragment)) errors.push(`${file}: missing fragment target #${fragment}`) @@ -56,10 +62,11 @@ for (const file of htmlFiles) { for (const [, href] of html.matchAll(/href="(\/[^"]*)"/g)) { if (href === '/') continue const pathname = href.split(/[?#]/, 1)[0] + const relative = pathname.replace(/^\//, '') let target if (pathname.startsWith('/src/')) target = `.${pathname}` else if (pathname.startsWith('/assets/') || pathname === '/favicon.ico') target = `public${pathname}` - else target = pathname.endsWith('/') ? join(generatedRoot, pathname, 'index.html') : join(generatedRoot, pathname) + else target = pathname.endsWith('/') ? join(generatedRoot, relative, 'index.html') : join(generatedRoot, relative) if (!(await exists(target))) errors.push(`${file}: broken internal link ${href}`) } @@ -71,6 +78,58 @@ for (const file of htmlFiles) { } } +const siteUrl = 'https://paulw.xyz' +const generatedFile = (pathname) => { + const relative = pathname.replace(/^\//, '') + return pathname.endsWith('/') ? join(generatedRoot, relative, 'index.html') : join(generatedRoot, relative) +} + +for (const name of ['feed.xml', 'sitemap.xml']) { + if (!(await exists(join(generatedRoot, name)))) errors.push(`missing ${name}`) +} + +if (await exists(join(generatedRoot, 'feed.xml'))) { + const feed = await readFile(join(generatedRoot, 'feed.xml'), 'utf8') + if (!feed.includes('([^<]+)<\/guid>/g)) { + const { pathname } = new URL(loc) + if (!(await exists(generatedFile(pathname)))) errors.push(`feed.xml: missing page for ${loc}`) + } +} + +if (await exists(join(generatedRoot, 'sitemap.xml'))) { + const sitemap = await readFile(join(generatedRoot, 'sitemap.xml'), 'utf8') + if (!sitemap.includes('http://www.sitemaps.org/schemas/sitemap/0.9')) + errors.push('sitemap.xml: missing urlset namespace') + + const locs = [...sitemap.matchAll(/([^<]+)<\/loc>/g)].map((match) => match[1]) + for (const loc of locs) { + const { origin, pathname } = new URL(loc) + if (origin !== siteUrl) errors.push(`sitemap.xml: unexpected origin ${loc}`) + if (!(await exists(generatedFile(pathname))) && pathname !== '/sitemap.xml') + errors.push(`sitemap.xml: missing file for ${loc}`) + } + + const locSet = new Set(locs) + for (const file of htmlFiles) { + if (file.endsWith('404.html') || noindexFiles.has(file)) continue + const relative = file.slice(generatedRoot.length + 1).replaceAll('\\', '/') + const pathname = relative === 'index.html' ? '/' : `/${relative.replace(/\/index\.html$/, '/')}` + const loc = `${siteUrl}${pathname}` + if (!locSet.has(loc)) errors.push(`sitemap.xml: missing ${loc}`) + } +} + +if (!(await exists('public/robots.txt'))) errors.push('missing public/robots.txt') + +try { + const tracked = execFileSync('git', ['ls-files', '-z'], { encoding: 'utf8' }) + for (const file of tracked.split('\0').filter(Boolean)) { + if (/(?:^|\/)\.[^/]*\.md$/.test(file)) errors.push(`${file}: hidden markdown must not be tracked`) + } +} catch {} + if (errors.length > 0) { console.error(errors.join('\n')) process.exitCode = 1 diff --git a/generator.js b/generator.js index 9011239..2f5a43f 100644 --- a/generator.js +++ b/generator.js @@ -5,8 +5,16 @@ import { escapeHtml, h, raw, render } from './html.js' import { marked } from 'marked' const siteName = 'PaulW.XYZ' +const siteUrl = 'https://paulw.xyz' const output = '.generated' const markdownDirs = ['notes', 'posts'] +const includeDrafts = process.argv.includes('--drafts') +const navLabels = { + notes: 'Notes', + posts: 'Posts', + about: 'About', + licenses: 'Licenses', +} const standalonePages = [ { source: 'README.md', path: 'about', title: 'About' }, @@ -56,7 +64,7 @@ const createSlugger = () => { } } -const pageShell = (title, body, path = '/', navTitle = '') => +const pageShell = (title, body, path = '/', navTitle = '', { draft = false } = {}) => `${render( h('html', { lang: 'en-US' }, [ h('head', {}, [ @@ -65,8 +73,15 @@ const pageShell = (title, body, path = '/', navTitle = '') => name: 'viewport', content: 'width=device-width, initial-scale=1', }), + draft ? h('meta', { name: 'robots', content: 'noindex, nofollow' }) : undefined, h('title', {}, [title ? `${title} | ${siteName}` : siteName]), h('link', { rel: 'icon', href: '/favicon.ico' }), + h('link', { + rel: 'alternate', + type: 'application/rss+xml', + title: `${siteName} feed`, + href: '/feed.xml', + }), h('link', { rel: 'stylesheet', href: '/src/global.css' }), h('script', { type: 'module', src: '/src/main.js' }), ]), @@ -81,10 +96,13 @@ const nav = (path, currentTitle = '') => { } const parts = path.split('/').filter(Boolean) const children = [h('a', { href: '/' }, [siteName])] - const sectionTitle = { notes: 'Notes', posts: 'Posts', about: 'About', licenses: 'Licenses' }[parts[0]] ?? parts[0] - if (parts.length >= 1) - children.push(' / ', parts.length > 1 ? h('a', { href: `/${parts[0]}/` }, [sectionTitle]) : sectionTitle) - if (parts.length > 1) children.push(' / ', currentTitle) + let href = '' + for (const [index, part] of parts.entries()) { + href += `/${part}` + const isLast = index === parts.length - 1 + const label = isLast && currentTitle ? currentTitle : (navLabels[part] ?? part) + children.push(' / ', isLast ? label : h('a', { href: `${href}/` }, [label])) + } return render(children) } @@ -107,9 +125,18 @@ const dateFor = async (file) => { } const readPages = async (dir) => { - const entries = await readdir(dir, { withFileTypes: true }) + let entries + try { + entries = await readdir(dir, { withFileTypes: true }) + } catch (error) { + if (error.code === 'ENOENT') return [] + throw error + } const markdownEntries = entries - .filter((entry) => entry.isFile() && entry.name.endsWith('.md') && !entry.name.startsWith('.')) + .filter((entry) => { + if (!entry.isFile() || !entry.name.endsWith('.md') || entry.name === '.md') return false + return includeDrafts || !entry.name.startsWith('.') + }) .sort((a, b) => a.name.localeCompare(b.name)) const pages = await Promise.all( markdownEntries.map(async (entry) => { @@ -117,16 +144,25 @@ const readPages = async (dir) => { const source = await readFile(file, 'utf8') const titleLine = source.split(/\r?\n/).find((line) => line.startsWith('# ')) if (!titleLine) return undefined + const draft = entry.name.startsWith('.') + const name = (draft ? entry.name.slice(1) : entry.name).slice(0, -3) + if (!name) return undefined return { - name: entry.name.slice(0, -3), + name, title: stripSidenotes(titleLine.slice(2).trim()), file, source, date: await dateFor(file), + draft, } }), ) - return pages.filter(Boolean) + const byName = new Map() + for (const page of pages.filter(Boolean)) { + const existing = byName.get(page.name) + if (!existing || (existing.draft && !page.draft)) byName.set(page.name, page) + } + return [...byName.values()].sort((a, b) => a.name.localeCompare(b.name)) } let nextHeadingId = createSlugger() @@ -268,6 +304,74 @@ const toc = (source) => { return renderItems(root.children) } +const excerpt = (source, max = 280) => { + const paragraph = marked + .lexer(withoutTitle(stripHeadingSidenotes(source))) + .find((token) => token.type === 'paragraph') + if (!paragraph) return '' + const text = stripSidenotes(headingLabel(paragraph.tokens ?? [{ text: paragraph.text ?? '' }])) + .replace(/\s+/g, ' ') + .trim() + if (text.length <= max) return text + const slice = text.slice(0, max) + const bound = slice.lastIndexOf(' ') + return `${(bound > 80 ? slice.slice(0, bound) : slice).trimEnd()}…` +} + +const toRssDate = (iso) => { + if (!iso) return '' + const date = new Date(iso) + return Number.isNaN(date.valueOf()) ? '' : date.toUTCString() +} + +const toSitemapDate = (iso) => (iso ? iso.slice(0, 10) : '') + +const renderFeed = (items) => { + const lastBuild = toRssDate(items.find((item) => item.date)?.date) || new Date().toUTCString() + const itemXml = items + .map((item) => { + const loc = `${siteUrl}/${item.dir}/${item.name}/` + const description = excerpt(item.source) + const pubDate = toRssDate(item.date) + return ` + ${escapeHtml(item.title)} + ${escapeHtml(loc)} + ${escapeHtml(loc)} + ${pubDate ? `${escapeHtml(pubDate)}` : ''} + ${escapeHtml(item.dir)} + ${description ? `${escapeHtml(description)}` : ''} + ` + }) + .join('\n ') + + return ` + + + ${escapeHtml(siteName)} + ${escapeHtml(`${siteUrl}/`)} + ${escapeHtml(`Notes and posts from ${siteName}`)} + en-US + + ${escapeHtml(lastBuild)} + ${itemXml} + + +` +} + +const renderSitemap = (urls) => ` + +${urls + .map(({ loc, lastmod }) => { + const modified = lastmod ? `\n ${escapeHtml(lastmod)}` : '' + return ` + ${escapeHtml(loc)}${modified} + ` + }) + .join('\n')} + +` + const writePage = async (path, html) => { const directory = join(output, path) await mkdir(directory, { recursive: true }) @@ -356,17 +460,20 @@ const renderArticle = (directory, page, navigationPath = `/${directory}/${page.n ? `` : '' const heading = titleHeading(page.title, `${directory}-heading`) + const draftNotice = page.draft + ? `

Draft — rename .${escapeHtml(page.name)}.md to ${escapeHtml(page.name)}.md to publish.

` + : '' const tableOfContents = toc(page.source) const contents = tableOfContents ? `
Contents${tableOfContents}
` : '' const article = `
${renderMarkdown(page.source)}
` - const body = `${heading}
${updated}${contents}${article}
` + const body = `${heading}
${draftNotice}${updated}${contents}${article}
` - return pageShell(page.title, body, navigationPath, page.title) + return pageShell(page.title, body, navigationPath, page.title, { draft: Boolean(page.draft) }) } -if (process.argv.includes('--clean')) await rm(output, { recursive: true, force: true }) +await rm(output, { recursive: true, force: true }) await mkdir(output, { recursive: true }) const sections = Object.fromEntries(await Promise.all(markdownDirs.map(async (dir) => [dir, await readPages(dir)]))) @@ -386,7 +493,7 @@ const standaloneLinks = standalonePages .join('') const recentPages = markdownDirs - .flatMap((dir) => sections[dir].map((page) => ({ ...page, dir }))) + .flatMap((dir) => sections[dir].filter((page) => !page.draft).map((page) => ({ ...page, dir }))) .sort((a, b) => b.date.localeCompare(a.date)) .slice(0, 5) @@ -401,13 +508,20 @@ await writeFile(join(output, '404.html'), renderNotFound()) for (const dir of markdownDirs) { const pages = sections[dir] - await writePage(dir, renderSection(dir, pages)) + await writePage( + dir, + renderSection( + dir, + pages.filter((page) => !page.draft), + ), + ) for (const page of pages) { await writePage(join(dir, page.name), renderArticle(dir, page)) } } +const generatedStandalone = [] for (const page of standalonePages) { const source = await readFile(page.source, 'utf8') const generatedPage = { @@ -416,5 +530,36 @@ for (const page of standalonePages) { source, date: await dateFor(page.source), } + generatedStandalone.push(generatedPage) await writePage(page.path, renderArticle(page.path, generatedPage, '/')) } + +const feedItems = markdownDirs + .flatMap((dir) => sections[dir].filter((page) => !page.draft).map((page) => ({ ...page, dir }))) + .sort((a, b) => b.date.localeCompare(a.date)) + +await writeFile(join(output, 'feed.xml'), renderFeed(feedItems)) + +const sitemapUrls = [ + { + loc: `${siteUrl}/`, + lastmod: toSitemapDate(feedItems[0]?.date), + }, + ...markdownDirs.map((dir) => ({ + loc: `${siteUrl}/${dir}/`, + lastmod: toSitemapDate( + [...sections[dir]].filter((page) => !page.draft).sort((a, b) => b.date.localeCompare(a.date))[0]?.date, + ), + })), + ...generatedStandalone.map((page) => ({ + loc: `${siteUrl}/${page.name}/`, + lastmod: toSitemapDate(page.date), + })), + ...feedItems.map((page) => ({ + loc: `${siteUrl}/${page.dir}/${page.name}/`, + lastmod: toSitemapDate(page.date), + })), + { loc: `${siteUrl}/feed.xml`, lastmod: toSitemapDate(feedItems[0]?.date) }, +] + +await writeFile(join(output, 'sitemap.xml'), renderSitemap(sitemapUrls)) diff --git a/package.json b/package.json index 48c60b6..fe9a629 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "generate": "node generator.js", "check": "node generator.js --clean && node check.js", "clean": "node clean.js", - "dev": "node generator.js && vite", + "dev": "node generator.js --drafts && vite", "build": "node generator.js --clean && node check.js && vite build", "preview": "vite preview" }, diff --git a/public/robots.txt b/public/robots.txt new file mode 100644 index 0000000..2029f08 --- /dev/null +++ b/public/robots.txt @@ -0,0 +1,4 @@ +User-agent: * +Allow: / + +Sitemap: https://paulw.xyz/sitemap.xml diff --git a/src/global.css b/src/global.css index eff39a7..abd43d0 100644 --- a/src/global.css +++ b/src/global.css @@ -787,6 +787,17 @@ ul li { margin: 0.5rem 0.75rem; } +.draft-banner { + margin: 0.5rem 0.75rem 0; + padding: 0.5rem 0.75rem; + border: 1px solid var(--main-border-color); + font-size: 1rem; +} + +.draft-banner code { + font-family: var(--monospace-font); +} + nav { padding: 0.25rem 0.75rem; font-size: 1.25rem; diff --git a/vite.config.js b/vite.config.js index fc3feaa..ae4e701 100644 --- a/vite.config.js +++ b/vite.config.js @@ -19,6 +19,21 @@ function generatedPages(directory) { return inputs } +function copyGeneratedFiles() { + const files = ['feed.xml', 'sitemap.xml'] + return { + name: 'copy-generated-files', + closeBundle() { + const dist = path.resolve('dist') + if (!fs.existsSync(dist)) return + for (const file of files) { + const source = path.resolve('.generated', file) + if (fs.existsSync(source)) fs.copyFileSync(source, path.join(dist, file)) + } + }, + } +} + function contentGenerator() { let timer let generator @@ -30,7 +45,7 @@ function contentGenerator() { return } - generator = spawn(process.execPath, ['generator.js'], { stdio: 'inherit' }) + generator = spawn(process.execPath, ['generator.js', '--drafts'], { stdio: 'inherit' }) generator.on('error', (error) => { console.error(`Content generation failed: ${error.message}`) }) @@ -77,7 +92,7 @@ function contentGenerator() { } export default defineConfig({ - plugins: [contentGenerator()], + plugins: [contentGenerator(), copyGeneratedFiles()], root: '.generated', publicDir: '../public', resolve: {