Add RSS, sitemap, and local dotted-file drafts.

check.js

@@ -1,3 +1,4 @@
+import { execFileSync } from 'node:child_process'
 import { readdir, readFile, stat } from 'node:fs/promises'
 import { join } from 'node:path'
 
@@ -24,6 +25,7 @@ const exists = async (path) => {
 
 const errors = []
 const htmlFiles = (await walk(generatedRoot)).filter((file) => file.endsWith('.html'))
+const noindexFiles = new Set()
 const clientSource = await readFile('src/main.js', 'utf8')
 const builtInPrismLanguages = new Set([
   'markup',
@@ -48,6 +50,10 @@ for (const file of htmlFiles) {
   if (duplicateIds.length > 0) errors.push(`${file}: duplicate IDs: ${[...new Set(duplicateIds)].join(', ')}`)
   if (!html.includes('<meta name="viewport"')) errors.push(`${file}: missing viewport metadata`)
   if (html.includes('<heading')) errors.push(`${file}: contains a nonstandard <heading> element`)
+  if (html.includes('name="robots"') && html.includes('noindex')) {
+    noindexFiles.add(file)
+    errors.push(`${file}: draft page must not be in production output`)
+  }
 
   for (const [, fragment] of html.matchAll(/href="#([^"]+)"/g)) {
     if (!ids.includes(fragment)) errors.push(`${file}: missing fragment target #${fragment}`)
@@ -56,10 +62,11 @@ for (const file of htmlFiles) {
   for (const [, href] of html.matchAll(/href="(\/[^"]*)"/g)) {
     if (href === '/') continue
     const pathname = href.split(/[?#]/, 1)[0]
+    const relative = pathname.replace(/^\//, '')
     let target
     if (pathname.startsWith('/src/')) target = `.${pathname}`
     else if (pathname.startsWith('/assets/') || pathname === '/favicon.ico') target = `public${pathname}`
-    else target = pathname.endsWith('/') ? join(generatedRoot, pathname, 'index.html') : join(generatedRoot, pathname)
+    else target = pathname.endsWith('/') ? join(generatedRoot, relative, 'index.html') : join(generatedRoot, relative)
     if (!(await exists(target))) errors.push(`${file}: broken internal link ${href}`)
   }
 
@@ -71,6 +78,58 @@ for (const file of htmlFiles) {
   }
 }
 
+const siteUrl = 'https://paulw.xyz'
+const generatedFile = (pathname) => {
+  const relative = pathname.replace(/^\//, '')
+  return pathname.endsWith('/') ? join(generatedRoot, relative, 'index.html') : join(generatedRoot, relative)
+}
+
+for (const name of ['feed.xml', 'sitemap.xml']) {
+  if (!(await exists(join(generatedRoot, name)))) errors.push(`missing ${name}`)
+}
+
+if (await exists(join(generatedRoot, 'feed.xml'))) {
+  const feed = await readFile(join(generatedRoot, 'feed.xml'), 'utf8')
+  if (!feed.includes('<rss version="2.0"')) errors.push('feed.xml: missing RSS 2.0 channel')
+  if (feed.includes('noindex')) errors.push('feed.xml: unexpected noindex marker')
+  for (const [, loc] of feed.matchAll(/<guid isPermaLink="true">([^<]+)<\/guid>/g)) {
+    const { pathname } = new URL(loc)
+    if (!(await exists(generatedFile(pathname)))) errors.push(`feed.xml: missing page for ${loc}`)
+  }
+}
+
+if (await exists(join(generatedRoot, 'sitemap.xml'))) {
+  const sitemap = await readFile(join(generatedRoot, 'sitemap.xml'), 'utf8')
+  if (!sitemap.includes('http://www.sitemaps.org/schemas/sitemap/0.9'))
+    errors.push('sitemap.xml: missing urlset namespace')
+
+  const locs = [...sitemap.matchAll(/<loc>([^<]+)<\/loc>/g)].map((match) => match[1])
+  for (const loc of locs) {
+    const { origin, pathname } = new URL(loc)
+    if (origin !== siteUrl) errors.push(`sitemap.xml: unexpected origin ${loc}`)
+    if (!(await exists(generatedFile(pathname))) && pathname !== '/sitemap.xml')
+      errors.push(`sitemap.xml: missing file for ${loc}`)
+  }
+
+  const locSet = new Set(locs)
+  for (const file of htmlFiles) {
+    if (file.endsWith('404.html') || noindexFiles.has(file)) continue
+    const relative = file.slice(generatedRoot.length + 1).replaceAll('\\', '/')
+    const pathname = relative === 'index.html' ? '/' : `/${relative.replace(/\/index\.html$/, '/')}`
+    const loc = `${siteUrl}${pathname}`
+    if (!locSet.has(loc)) errors.push(`sitemap.xml: missing ${loc}`)
+  }
+}
+
+if (!(await exists('public/robots.txt'))) errors.push('missing public/robots.txt')
+
+try {
+  const tracked = execFileSync('git', ['ls-files', '-z'], { encoding: 'utf8' })
+  for (const file of tracked.split('\0').filter(Boolean)) {
+    if (/(?:^|\/)\.[^/]*\.md$/.test(file)) errors.push(`${file}: hidden markdown must not be tracked`)
+  }
+} catch {}
+
 if (errors.length > 0) {
   console.error(errors.join('\n'))
   process.exitCode = 1

generator.js

@@ -5,8 +5,16 @@ import { escapeHtml, h, raw, render } from './html.js'
 import { marked } from 'marked'
 
 const siteName = 'PaulW.XYZ'
+const siteUrl = 'https://paulw.xyz'
 const output = '.generated'
 const markdownDirs = ['notes', 'posts']
+const includeDrafts = process.argv.includes('--drafts')
+const navLabels = {
+  notes: 'Notes',
+  posts: 'Posts',
+  about: 'About',
+  licenses: 'Licenses',
+}
 
 const standalonePages = [
   { source: 'README.md', path: 'about', title: 'About' },
@@ -56,7 +64,7 @@ const createSlugger = () => {
   }
 }
 
-const pageShell = (title, body, path = '/', navTitle = '') =>
+const pageShell = (title, body, path = '/', navTitle = '', { draft = false } = {}) =>
   `<!DOCTYPE html>${render(
     h('html', { lang: 'en-US' }, [
       h('head', {}, [
@@ -65,8 +73,15 @@ const pageShell = (title, body, path = '/', navTitle = '') =>
           name: 'viewport',
           content: 'width=device-width, initial-scale=1',
         }),
+        draft ? h('meta', { name: 'robots', content: 'noindex, nofollow' }) : undefined,
         h('title', {}, [title ? `${title} | ${siteName}` : siteName]),
         h('link', { rel: 'icon', href: '/favicon.ico' }),
+        h('link', {
+          rel: 'alternate',
+          type: 'application/rss+xml',
+          title: `${siteName} feed`,
+          href: '/feed.xml',
+        }),
         h('link', { rel: 'stylesheet', href: '/src/global.css' }),
         h('script', { type: 'module', src: '/src/main.js' }),
       ]),
@@ -81,10 +96,13 @@ const nav = (path, currentTitle = '') => {
   }
   const parts = path.split('/').filter(Boolean)
   const children = [h('a', { href: '/' }, [siteName])]
-  const sectionTitle = { notes: 'Notes', posts: 'Posts', about: 'About', licenses: 'Licenses' }[parts[0]] ?? parts[0]
-  if (parts.length >= 1)
-    children.push(' / ', parts.length > 1 ? h('a', { href: `/${parts[0]}/` }, [sectionTitle]) : sectionTitle)
-  if (parts.length > 1) children.push(' / ', currentTitle)
+  let href = ''
+  for (const [index, part] of parts.entries()) {
+    href += `/${part}`
+    const isLast = index === parts.length - 1
+    const label = isLast && currentTitle ? currentTitle : (navLabels[part] ?? part)
+    children.push(' / ', isLast ? label : h('a', { href: `${href}/` }, [label]))
+  }
   return render(children)
 }
 
@@ -107,9 +125,18 @@ const dateFor = async (file) => {
 }
 
 const readPages = async (dir) => {
-  const entries = await readdir(dir, { withFileTypes: true })
+  let entries
+  try {
+    entries = await readdir(dir, { withFileTypes: true })
+  } catch (error) {
+    if (error.code === 'ENOENT') return []
+    throw error
+  }
   const markdownEntries = entries
-    .filter((entry) => entry.isFile() && entry.name.endsWith('.md') && !entry.name.startsWith('.'))
+    .filter((entry) => {
+      if (!entry.isFile() || !entry.name.endsWith('.md') || entry.name === '.md') return false
+      return includeDrafts || !entry.name.startsWith('.')
+    })
     .sort((a, b) => a.name.localeCompare(b.name))
   const pages = await Promise.all(
     markdownEntries.map(async (entry) => {
@@ -117,16 +144,25 @@ const readPages = async (dir) => {
       const source = await readFile(file, 'utf8')
       const titleLine = source.split(/\r?\n/).find((line) => line.startsWith('# '))
       if (!titleLine) return undefined
+      const draft = entry.name.startsWith('.')
+      const name = (draft ? entry.name.slice(1) : entry.name).slice(0, -3)
+      if (!name) return undefined
       return {
-        name: entry.name.slice(0, -3),
+        name,
         title: stripSidenotes(titleLine.slice(2).trim()),
         file,
         source,
         date: await dateFor(file),
+        draft,
       }
     }),
   )
-  return pages.filter(Boolean)
+  const byName = new Map()
+  for (const page of pages.filter(Boolean)) {
+    const existing = byName.get(page.name)
+    if (!existing || (existing.draft && !page.draft)) byName.set(page.name, page)
+  }
+  return [...byName.values()].sort((a, b) => a.name.localeCompare(b.name))
 }
 
 let nextHeadingId = createSlugger()
@@ -268,6 +304,74 @@ const toc = (source) => {
   return renderItems(root.children)
 }
 
+const excerpt = (source, max = 280) => {
+  const paragraph = marked
+    .lexer(withoutTitle(stripHeadingSidenotes(source)))
+    .find((token) => token.type === 'paragraph')
+  if (!paragraph) return ''
+  const text = stripSidenotes(headingLabel(paragraph.tokens ?? [{ text: paragraph.text ?? '' }]))
+    .replace(/\s+/g, ' ')
+    .trim()
+  if (text.length <= max) return text
+  const slice = text.slice(0, max)
+  const bound = slice.lastIndexOf(' ')
+  return `${(bound > 80 ? slice.slice(0, bound) : slice).trimEnd()}…`
+}
+
+const toRssDate = (iso) => {
+  if (!iso) return ''
+  const date = new Date(iso)
+  return Number.isNaN(date.valueOf()) ? '' : date.toUTCString()
+}
+
+const toSitemapDate = (iso) => (iso ? iso.slice(0, 10) : '')
+
+const renderFeed = (items) => {
+  const lastBuild = toRssDate(items.find((item) => item.date)?.date) || new Date().toUTCString()
+  const itemXml = items
+    .map((item) => {
+      const loc = `${siteUrl}/${item.dir}/${item.name}/`
+      const description = excerpt(item.source)
+      const pubDate = toRssDate(item.date)
+      return `<item>
+      <title>${escapeHtml(item.title)}</title>
+      <link>${escapeHtml(loc)}</link>
+      <guid isPermaLink="true">${escapeHtml(loc)}</guid>
+      ${pubDate ? `<pubDate>${escapeHtml(pubDate)}</pubDate>` : ''}
+      <category>${escapeHtml(item.dir)}</category>
+      ${description ? `<description>${escapeHtml(description)}</description>` : ''}
+    </item>`
+    })
+    .join('\n    ')
+
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
+  <channel>
+    <title>${escapeHtml(siteName)}</title>
+    <link>${escapeHtml(`${siteUrl}/`)}</link>
+    <description>${escapeHtml(`Notes and posts from ${siteName}`)}</description>
+    <language>en-US</language>
+    <atom:link href="${escapeHtml(`${siteUrl}/feed.xml`)}" rel="self" type="application/rss+xml"/>
+    <lastBuildDate>${escapeHtml(lastBuild)}</lastBuildDate>
+    ${itemXml}
+  </channel>
+</rss>
+`
+}
+
+const renderSitemap = (urls) => `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${urls
+  .map(({ loc, lastmod }) => {
+    const modified = lastmod ? `\n    <lastmod>${escapeHtml(lastmod)}</lastmod>` : ''
+    return `  <url>
+    <loc>${escapeHtml(loc)}</loc>${modified}
+  </url>`
+  })
+  .join('\n')}
+</urlset>
+`
+
 const writePage = async (path, html) => {
   const directory = join(output, path)
   await mkdir(directory, { recursive: true })
@@ -356,17 +460,20 @@ const renderArticle = (directory, page, navigationPath = `/${directory}/${page.n
     ? `<time class="last-updated" datetime="${escapeHtml(page.date)}" data-value="${escapeHtml(page.date)}">Last updated: ${escapeHtml(page.date)}</time>`
     : ''
   const heading = titleHeading(page.title, `${directory}-heading`)
+  const draftNotice = page.draft
+    ? `<p class="draft-banner" role="status">Draft — rename <code>.${escapeHtml(page.name)}.md</code> to <code>${escapeHtml(page.name)}.md</code> to publish.</p>`
+    : ''
   const tableOfContents = toc(page.source)
   const contents = tableOfContents
     ? `<section class="block toc"><details open><summary>Contents</summary>${tableOfContents}</details></section><hr>`
     : ''
   const article = `<section class="block">${renderMarkdown(page.source)}</section>`
-  const body = `${heading}<main class="article-page ${directory}-page container">${updated}${contents}${article}</main>`
+  const body = `${heading}<main class="article-page ${directory}-page container">${draftNotice}${updated}${contents}${article}</main>`
 
-  return pageShell(page.title, body, navigationPath, page.title)
+  return pageShell(page.title, body, navigationPath, page.title, { draft: Boolean(page.draft) })
 }
 
-if (process.argv.includes('--clean')) await rm(output, { recursive: true, force: true })
+await rm(output, { recursive: true, force: true })
 await mkdir(output, { recursive: true })
 const sections = Object.fromEntries(await Promise.all(markdownDirs.map(async (dir) => [dir, await readPages(dir)])))
 
@@ -386,7 +493,7 @@ const standaloneLinks = standalonePages
   .join('')
 
 const recentPages = markdownDirs
-  .flatMap((dir) => sections[dir].map((page) => ({ ...page, dir })))
+  .flatMap((dir) => sections[dir].filter((page) => !page.draft).map((page) => ({ ...page, dir })))
   .sort((a, b) => b.date.localeCompare(a.date))
   .slice(0, 5)
 
@@ -401,13 +508,20 @@ await writeFile(join(output, '404.html'), renderNotFound())
 
 for (const dir of markdownDirs) {
   const pages = sections[dir]
-  await writePage(dir, renderSection(dir, pages))
+  await writePage(
+    dir,
+    renderSection(
+      dir,
+      pages.filter((page) => !page.draft),
+    ),
+  )
 
   for (const page of pages) {
     await writePage(join(dir, page.name), renderArticle(dir, page))
   }
 }
 
+const generatedStandalone = []
 for (const page of standalonePages) {
   const source = await readFile(page.source, 'utf8')
   const generatedPage = {
@@ -416,5 +530,36 @@ for (const page of standalonePages) {
     source,
     date: await dateFor(page.source),
   }
+  generatedStandalone.push(generatedPage)
   await writePage(page.path, renderArticle(page.path, generatedPage, '/'))
 }
+
+const feedItems = markdownDirs
+  .flatMap((dir) => sections[dir].filter((page) => !page.draft).map((page) => ({ ...page, dir })))
+  .sort((a, b) => b.date.localeCompare(a.date))
+
+await writeFile(join(output, 'feed.xml'), renderFeed(feedItems))
+
+const sitemapUrls = [
+  {
+    loc: `${siteUrl}/`,
+    lastmod: toSitemapDate(feedItems[0]?.date),
+  },
+  ...markdownDirs.map((dir) => ({
+    loc: `${siteUrl}/${dir}/`,
+    lastmod: toSitemapDate(
+      [...sections[dir]].filter((page) => !page.draft).sort((a, b) => b.date.localeCompare(a.date))[0]?.date,
+    ),
+  })),
+  ...generatedStandalone.map((page) => ({
+    loc: `${siteUrl}/${page.name}/`,
+    lastmod: toSitemapDate(page.date),
+  })),
+  ...feedItems.map((page) => ({
+    loc: `${siteUrl}/${page.dir}/${page.name}/`,
+    lastmod: toSitemapDate(page.date),
+  })),
+  { loc: `${siteUrl}/feed.xml`, lastmod: toSitemapDate(feedItems[0]?.date) },
+]
+
+await writeFile(join(output, 'sitemap.xml'), renderSitemap(sitemapUrls))

package.json

@@ -6,7 +6,7 @@
     "generate": "node generator.js",
     "check": "node generator.js --clean && node check.js",
     "clean": "node clean.js",
-    "dev": "node generator.js && vite",
+    "dev": "node generator.js --drafts && vite",
     "build": "node generator.js --clean && node check.js && vite build",
     "preview": "vite preview"
   },

public/robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /
+
+Sitemap: https://paulw.xyz/sitemap.xml

src/global.css

@@ -787,6 +787,17 @@ ul li {
   margin: 0.5rem 0.75rem;
 }
 
+.draft-banner {
+  margin: 0.5rem 0.75rem 0;
+  padding: 0.5rem 0.75rem;
+  border: 1px solid var(--main-border-color);
+  font-size: 1rem;
+}
+
+.draft-banner code {
+  font-family: var(--monospace-font);
+}
+
 nav {
   padding: 0.25rem 0.75rem;
   font-size: 1.25rem;

vite.config.js

@@ -19,6 +19,21 @@ function generatedPages(directory) {
   return inputs
 }
 
+function copyGeneratedFiles() {
+  const files = ['feed.xml', 'sitemap.xml']
+  return {
+    name: 'copy-generated-files',
+    closeBundle() {
+      const dist = path.resolve('dist')
+      if (!fs.existsSync(dist)) return
+      for (const file of files) {
+        const source = path.resolve('.generated', file)
+        if (fs.existsSync(source)) fs.copyFileSync(source, path.join(dist, file))
+      }
+    },
+  }
+}
+
 function contentGenerator() {
   let timer
   let generator
@@ -30,7 +45,7 @@ function contentGenerator() {
       return
     }
 
-    generator = spawn(process.execPath, ['generator.js'], { stdio: 'inherit' })
+    generator = spawn(process.execPath, ['generator.js', '--drafts'], { stdio: 'inherit' })
     generator.on('error', (error) => {
       console.error(`Content generation failed: ${error.message}`)
     })
@@ -77,7 +92,7 @@ function contentGenerator() {
 }
 
 export default defineConfig({
-  plugins: [contentGenerator()],
+  plugins: [contentGenerator(), copyGeneratedFiles()],
   root: '.generated',
   publicDir: '../public',
   resolve: {