zvava.org/make.js

523 lines
22 KiB
JavaScript

import * as std from "std"
import * as os from "os"
// requires a unix system to run
// main()
// → fetchTemplates
// → fetchWiki
// → generateTemplates
// · fill out dynamic templates and cache
// → generateGemini
// · write gemini output files
// → generateHTML
// · translate gemini files into html & write
// → generateAss
// · write a list of non-stub articles
//
// parseGemini()
// → converts gemtext into html
function runCommand(cmd, args = []) {
let pipe = os.pipe() // open pipe
let pid = os.exec([ ...cmd.split(/\s+/g), ...args ], { stdout: pipe[1] }) // exec command and pipe stdout to pipe
os.close(pipe[1]) // close pipe for writing
let file = std.fdopen(pipe[0], 'r') // open pipe to read
const result = file.readAsString().trim() // capture result
file.close() // close pipe for reading
return result
}
function removeFile(file, recursive = false) {
let cmd = !recursive ? ["rm"] : ["rm", "-r"] // if recursive use -r
Array.isArray(file) ? cmd.push(...file) : cmd.push(file) // push array of files or a single file
return os.exec(cmd)
}
function copyFile(from, to, recursive = false) {
let cmd = !recursive ? ["cp"] : ["cp", "-r"] // if recursive use -r
cmd.push(from, to) // push from and to
return os.exec(cmd)
}
function readDir(path, reject = (e) => { throw new Error(e) }, resolve = r => r) {
let [ files, err ] = os.readdir(path)
if (err) return reject(err)
files = files.filter(x => x != "." && x != "..") // remove relative directories
// check subdirectories
let subdirs = []
files.forEach(d => readDir(path + "/" + d, e => {}, sd => {
subdirs.push(d)
sd.forEach(f => files.push(d + "/" + f))
}))
return resolve(files.filter(x => !subdirs.includes(x))) // remove subdirectories
}
function readFile(path, reject = (e) => { throw new Error(e) }, resolve = r => r) {
let _f = std.open(path, "r")
if (_f == null || _f.error()) return reject("error reading " + path)
let data = _f.readAsString().replace(/\r/g, "") // eliminate crlf
if (path.endsWith(".json")) data = JSON.parse(data) // parse json if necessary
_f.close()
return resolve(data)
}
function prependRelevantEmoji(x) {
let e = templates["categories.json"][x] || templates["categories.json"]["unknown"] || ""
return e.length > 0 ? e + " " + x : x
}
function sortByModified(a, b) {
return new Date(b.modified.replace(/\//g, "-")) - new Date(a.modified.replace(/\//g, "-")) }
function stringifyDate(date) {
let d = new Date(date)
let month = (d.getUTCMonth() + 1).toString()
if (month.length == 1) month = "0" + month
let day = d.getUTCDate().toString()
if (day.length == 1) day = "0" + day
return `${d.getUTCFullYear()}/${month}/${day}`
}
function useTemplate(template, fields, values) {
let x = templates[template]
fields.forEach((f, i) => x = x.replace("{"+f+"}", values ? values[i] : templates[f]))
return x
}
function escapeForScript(content) {
return content.replace(/(?<!\\)(`|\$\()/g, "\\$1").trim()
}
/** @type {string[]} */
let categories = {}
/** @type {Object.<string, string>} */
let tmpls = {}
/** @type {Object.<string, string>} */
let templates
let pages = []
let ctgs = []
main()
// ...
function main() {
if (os.platform == "win32" || os.platform == "js")
return print(`make: cannot run on this platform (${os.platform})`)
print("\x1b[90m->\x1b[0m updating resources...")
// clear build directory
removeFile(["out"], true)
os.mkdir("out")
// gemini output directory
os.mkdir("out/gemini")
os.mkdir("out/gemini/wiki")
os.mkdir("out/gemini/wiki/category")
// html output directory
os.mkdir("out/www")
os.mkdir("out/www/wiki")
os.mkdir("out/www/wiki/category")
// update gemini resources
copyFile("src/images/", "out/gemini/images/", true)
copyFile("src/favicon.txt", "out/gemini/favicon.txt")
// update web resources
copyFile("src/images/", "out/www/images/", true)
copyFile("src/zvava.css", "out/www/zvava.css")
copyFile("src/zvava.js", "out/www/zvava.js")
copyFile("src/zvava.js.css", "out/www/zvava.js.css")
fetchTemplates()
}
// read all of the templates
function fetchTemplates() {
print("\x1b[90m->\x1b[0m gathering templates...")
let err, temps = readDir("src/templates", e => err = e)
if (err) return print(err)
let count = 0
// iterate over and read asynchronously
temps.forEach((filename) => new Promise((resolve, reject) => {
tmpls[filename] = readFile("src/templates/" + filename, reject);
// proceed
resolve(); if (++count == temps.length) {
templates = { ...tmpls }
fetchWiki()
}
}).catch(e => print("\nmake.js: gathering template:", filename, e))) // handle errors
}
// read and parse all of the wiki pages
function fetchWiki() {
print("\x1b[90m->\x1b[0m gathering wiki pages...")
let err, _wiki = readDir("src/wiki", e => err = e)
if (err) return print(err)
let count = 0
// remove file extensions, iterate over and process asynchronously
_wiki.map(x => x.replace(/\.\w*?$/, "")).forEach((_page) => new Promise((resolve, reject) => {
let page = _page.replace(/^[\w\/]*\//, "")
let data = readFile("src/wiki/" + _page + ".gmi", reject)
let metadata = { page: page, content: data }
// extract title
metadata["title"] = data.substring(2, data.indexOf("\n"))
// extract metadata
let _m = data.indexOf("```") + 4 // start of metadata
data.substring(_m, data.indexOf("```", _m) - 1).split(/\n+/) // isolate metadata and split by newlines
// split lines by key/value, and iterate over metadata fields
.map(x => x.split(/\s+/)).forEach((x) => {
let key = x.shift(); metadata[key] = // get key
key == "category" ? x.map(y => y.replace(",", "")) : x.join(" ") // get value
})
// ensure there is a modified field
if (!metadata["modified"]) metadata["modified"] = metadata["created"]
// extract thumbnail
if (data.substring(0, _m - 4).includes("\n=>")) {
let start = data.indexOf("=>") + 3, end = data.indexOf("\n", start)
let potentialThumbnail = data.substring(start, end)
if (!potentialThumbnail.includes(".mp4")) metadata["thumbnail"] = potentialThumbnail
}
// collect unique categories
if (metadata["category"]) metadata["category"].forEach(c =>
!ctgs.includes(c) && ctgs.push(c))
pages.push(metadata); resolve()
// check if done and proceed
if (++count == _wiki.length) { // length minus . & ..
pages = pages.sort(sortByModified) // sort pages
generateTemplates() } // proceed
}).catch(e => print("\nmake.js: parsing page:", page, e))) // handle errors
}
// parse templates
function generateTemplates() {
print("\x1b[90m->\x1b[0m generating templates...")
let commits = runCommand("git rev-list --all --count"),
bootDate = new Date(runCommand("node -p", [`new Date('${runCommand("uptime -s")}')`])),
date = new Date()
// → {wiki_recent} template
let _books = ["📕", "📗", "📘", "📙", "📓"].sort(() => Math.random() - .5) // random book emojis
// remove stubs, get first five pages, and render template
templates["wiki_recent"] = pages.filter(page => !page.category.includes("stub")).slice(0, 5).map((p, i) => {
let book = _books[i] // get random book emoji
let category = prependRelevantEmoji(p.category[0]) // get category emoji
// render
return `=> /wiki/${p.page}.ln ${book} wiki/${p.title}`
+ "\n```\n " + `[${p.modified}] [${category}]` + "\n```"
}).join("\n") // stringify
// → {html_wiki_recent} template
// remove stubs, get first five pages, and render template
templates["html_wiki_recent"] = pages.filter(page => !page.category.includes("stub")).slice(0, 5).map((p, i) => {
let category = prependRelevantEmoji(p.category[0]).split(" ")[0] // get category emoji
// render
return `<a class="sidebar__link" title="${category} ${p.title}" href="/wiki/${p.page}.html">${category} ${p.title}<br><pre> ${p.modified}</pre></a>`
}).join("\n") // stringify
// → {wiki_all} template
// remove stubs and render template
templates["wiki_all"] = pages.filter(page => !page.category.includes("stub")).map(p => {
let category = p.category.map(prependRelevantEmoji).join(", ") // get category emojis
return `=> /wiki/${p.page}.ln ${p.title}` +
"\n```\n " + `[${p.modified}] [${category}]` + "\n```"
}).join("\n") // stringify
// → {wiki_categories} template
// ignore stubs and render template
templates["wiki_categories"] = ctgs.filter(ctg => ctg != "stub")
.map(ctg => `=> /wiki/category/${ctg}.ln ${prependRelevantEmoji(ctg)}`)
.join("\n") // stringify
// → {wiki_categories__meta} template
templates["wiki_categories__meta"] = "```\ntotal " + (ctgs.length - 1) + "\n```\n"
// → {wiki_category__[category]} templates
ctgs.forEach(c => {
let ps = pages.filter(page => page.category.includes(c)).map(p =>
`=> ${p.thumbnail || (p.category.includes("video") ? "/images/t/no-category-video.png" : "/images/t/no-category.png")}\n` +
`=> /wiki/${p.page}.ln ${p.title}`)
let catMeta = "```\nitems " + ps.length + "\n```\n", _cb = ""
let catBody = templates["categories/" + c + ".gmi"] || ""
// put meta within body, if possible
if ((_cb = catBody.replace("{category_meta}\n", catMeta)) != catBody) catMeta = "", catBody = _cb
templates["wiki_category__" + c] = catMeta + catBody + "\n" + ps.join("\n\n")
})
// → {wiki_pinned} and {html_wiki_pinned}
templates["wiki_pinned"] = templates["pinned-pages.gmi"]
templates["html_wiki_pinned"] = parseGemini("", templates["wiki_pinned"]).replace(/<p>|<\/p>|<br>/g, "")
.replace(/<a/g, "<div><a class=\"sidebar__link\"").replace(/<\/a>/g, "\n<pre></pre></a></div>")
// → {html_filter} template
templates["html_filter__inputs"] = '<input type="radio" name="category" id="all" checked>\n' +
ctgs.filter(x => x != "stub").map(x => `<input type="radio" name="category" id="${x}">`).join("\n")
templates["html_filter__labels"] = `<label class="all-label" for="all">${prependRelevantEmoji("all")}</label>\n` +
ctgs.filter(x => x != "stub").map(x => `<label class="${x}-label" for="${x}">${prependRelevantEmoji(x)}</label>`).join("\n")
templates["html_filter__selected"] = '#all:checked ~ details .all-label::before,\n' +
ctgs.filter(x => x != "stub").map(x => `#${x}:checked ~ details .${x}-label::before`).join(",\n")
templates["html_filter__selected_collapsed"] = ctgs.filter(x => x != "stub").map(x =>
`#${x}:checked ~ details summary::after { content: " (${x})"; }`).join("\n")
templates["html_filter__shown"] = ctgs.filter(x => x != "stub").map(x =>
`#${x}:checked ~ [data-category~="${x}"]`).join(",\n")
templates["html_filter"] = useTemplate("filter.html", ["html_filter__inputs", "html_filter__labels", "html_filter__selected", "html_filter__selected_collapsed", "html_filter__shown"])
// → {build_info_summary} template
templates["build_info_summary"] = `commit no. ${commits} built on ${stringifyDate(date)}`
// → {build_info} template
templates["build_info"] =
`# of commits ${commits}\n` +
`# of pages ${pages.length}\n` +
`last built on ${date.toUTCString().replace("GMT", "UTC")}`
// → {gmi_webrings} template (remove trailing newline)
templates["gmi_webrings"] = templates["webrings.gmi"].substring(0, templates["webrings.gmi"].length - 1)
// → {*_buttons} templates
templates["gmi_buttons"] = templates["buttons.gmi"].replace(/\.(gif|png)$/gm, "")
templates["html_buttons"] = templates["buttons.gmi"].replace(
/=> ((https:\/\/|#)([-\w\.]+).*?|) (.+)(\.\w+)/g,
'<a class="nob4" href="$1" title="$4"><img src="/images/buttons/$3$5" alt="$4\'s badge"></a>')
generateGemini() // proceed
}
// parse gemtext
function generateGemini() {
print("\x1b[90m->\x1b[0m generating gemini site...")
let files = {}
// use templates
// → index
files["index"] = useTemplate("index.gmi", [ "wiki_recent", "build_info_summary", "wiki_pinned" ])
// → stats
files["stats"] = useTemplate("stats.gmi", [ "build_info" ])
// → wiki/category/index
files["wiki/category/index"] = useTemplate("category-index.gmi", [ "wiki_categories", "wiki_categories__meta" ])
// → wiki/category/*
ctgs.forEach(c =>
files["wiki/category/" + c] = useTemplate("category-page.gmi",
[ "category_title", "wiki_category" ], [ c, templates["wiki_category__" + c] ]))
// → wiki/index
files["wiki/index"] = useTemplate("wiki-index.gmi", [ "wiki_all" ])
// → wiki/*
pages.forEach(p =>
files["wiki/" + p.page] = useTemplate("wiki-page.gmi", [ "content" ], [ p.content ]))
let count = 0
// iterate over each page and render final gemtext
let _files = Object.keys(files); _files.forEach((f) => new Promise((resolve, reject) => {
let content = files[f]
.replace(/{html[a-z_]*}\n/g, "") // remove html-only templates
.replace(/{br}\n/g, "") // remove html-only newlines
.replace(/{gmi_buttons}/g, templates["gmi_buttons"]) // buttons without images
.replace(/{gmi_webrings}/g, templates["gmi_webrings"]) // plain webrings
.replace(/(\w*)\.(png|jpg) (thumbnail|cover|image)/gi, "$1.$2 $3 ($1.$2)") // add filenames to thumbnails
.replace(/stay:\/\//g, "gemini://") // replace ambiguous protocols
.replace(/\.ln/g, ".gmi") // replace ambiguous links
if (f == "stats") { // statistics page cgi
content = templates["misc/stats.sh"].replace("{include_page}", escapeForScript(content))
} else if (f == "index" || f == "wiki/index") { // sneaky viewcounter
content = templates["misc/viewcounter.sh"].replace("{include_page}", escapeForScript(content))
} else if (f.startsWith("wiki/")) { // embedding viewcounter
content = templates["misc/wiki-viewcounter.sh"].replace("{include_page}", escapeForScript(content))
}
// open file, handle errors, write, and update status indicator
let _f = std.open("out/gemini/" + f + ".gmi", "w")
if (_f.error()) reject(_f.error())
_f.puts(content); _f.close(); resolve()
std.printf(`\r\x1b[32m-->\x1b[0m wrote gemini page ${count + 1}/${_files.length}`)
if (++count == _files.length) { // if all pages have been written
print(); generateHTML(files) } // proceed
}).catch(e => print("\nmake.js: writing page:", f, e))) // handle errors
}
// convert the articles into html
function generateHTML(files) {
print("\x1b[90m->\x1b[0m generating html site...")
files["sidebar.html"] = useTemplate("misc/sidebar.html", [ "html_wiki_pinned", "html_wiki_recent" ])
files["stats"] = files["stats"] // add graph to stats page
.replace("=> https://zvava.org/stats.html requires javascript to work :l", "{stats}")
files["wiki/about-site"] = files["wiki/about-site"] // alternate gemini links to https on about page
.replace("=> https://zvava.org 🕸️ view html version", "=> gemini://zvava.org 🚀 view gemini version")
let count = 0
// iterate over each page and render final html
let _files = Object.keys(files); _files.forEach((f) => new Promise((resolve, reject) => {
let output;
if (f.endsWith(".html")) {
output = files[f]
f = f.split(".")[0]
} else {
// get title and use <head> template
let title = `/${f} @ zvava.org`, isCategoryPage = f.startsWith("wiki/category/")
if (f == "index") title = "zvava.org"
else if (f == "wiki/index") title = "/wiki/ @ zvava.org"
else if (f == "wiki/category/index") title = "categories @ zvava.org"
else if (isCategoryPage) title = f.substring(14) + " @ zvava.org"
output = templates["head.html"].replace("{title}", title)
if (isCategoryPage) output = output.replace("<div class=\"wrap\">", `<div class="wrap wrap__category">`)
output = parseGemini(output, files[f], (l) => {
if (f == "index" && /href="https:\/\/(mk\.catgirlsfor\.science|git\.zvava\.org|www\.buymeacoffee\.com|matrix\.to)/.test(l))
l = l.replace(">", " rel=\"me\">") // add rel=me attribute to some links
if (f == "wiki/index" && /<a href="\/wiki\/(?!category).+"/.test(l)) { // to append data-category attribute for filters
let start = l.indexOf("/wiki/") + 6, end = l.indexOf(".html", start)
let page = pages.find(x => x.page == l.substring(start, end)) // get linked page
l = l.replace(">", ` data-category="${page ? page.category.join(" ") : "⚠️"}">`) // add
}
return l
})
}
if (f == "index") { // home page
output = output.replace("{html_buttons}", templates["html_buttons"]) // buttons!!
.replace("{html_webrings}", templates["webrings.html"]) // fancy webrings
.replace("{html_color}", templates["color.html"]) // hue-server client
// add viewcounter script that doesn't display the count
+ templates["misc/viewcounter.html"].replace("{url}", "/" + f + ".html")
} else if (f == "stats") { // statistics page
output = output // add statistics display + counter
+ templates["stats.html"]
+ templates["misc/viewcounter.html"].replace("{url}", "/" + f + ".html")
} else if (f == "wiki/index") { // wiki home page
// use filter template
output = output.replace("<p>{html_filter}<br>\n", templates["html_filter"] + "<p>")
// pass down data-category attribute from <a> to <p>s and <pre>s
.replace(/<p><a (href=".+?") (data-category=".+?")>(.+?)<\/a><\/p>\n<pre>/gm, "<p $2><a $1>$3</a></p>\n<pre $2>")
// add viewcounter script that doesn't display the count
output += templates["misc/viewcounter.html"]
.replace("{url}", "/" + f + ".html")
} else if (f.startsWith("wiki/")) { // any wiki page
output += templates["misc/wiki-viewcounter.html"] // add viewcounter script to wiki pages that displays the count
.replace("{url}", "/" + f + ".html")
}
output = output // final overrides
.replace(/{html_sbcont_start}(<br>)?/g, "<div class=\"sidebar-content\">")
.replace(/{html_sbcont_end}(<br>)?/g, "</div>")
let _f = std.open("out/www/" + f + ".html", "w")
if (_f.error()) reject(_f.error())
_f.puts(output + "</div></body>\n</html>\n")
_f.close(); resolve()
// update terminal readout
std.printf(`\r\x1b[32m-->\x1b[0m wrote html page ${count + 1}/${_files.length}`)
// if all pages have been written
if (++count == _files.length) {
print(); generateAss() }
}).catch(e => print("\nmake.js: writing page:", f, e)))
}
function generateAss() {
console.log("\x1b[90m->\x1b[0m generating feed.ass...")
let assEntries = "# Actually Simple Syndication - https://tilde.town/~dzwdz/ass/\n"
// iterate over pages that aren't stubs
+ pages.filter(x => x.category[0] != "stub").map(page => {
let date = page.modified.replace(/\//g, "-") // change date format slightly
return `${date} protocol://zvava.org/wiki/${page.page}.ln ${page.title}`
}).join("\n") + "\n" // combine into one
let fg = std.open("out/gemini/feed.ass", "w") // write gemini .ass
fg.puts(assEntries.replace(/protocol:\/\//g, "gemini://").replace(/\.ln\t/g, ".gmi\t")); fg.close()
let fw = std.open("out/www/feed.ass", "w") // write http .ass
fw.puts(assEntries.replace(/protocol:\/\//g, "https://").replace(/\.ln\t/g, ".html\t")); fw.close()
print("\x1b[32m-->\x1b[0m generated feed.ass")
print("\r\x1b[32m-->\x1b[0m finished make script")
}
//
function parseGemini(_output, file, hooks = (l) => l) {
let output = _output
let _c = file
.replace(/{gmi[a-z_]*}\n/g, "") // remove gmi-only templates
.replace(/\.ln/g, ".html") // replace ambiguous links
.replace(/stay:\/\//g, "https://") // replace ambiguous protocols
.split("\n```\n") // split into variable for optimized access to .length
_c.forEach((x, i) => {
// if file contains a code block and you are currently in one
if (_c.length > 1 && i % 2 !== 0)
return output += x + "</pre>\n"
// parse remaining content
output += x.split(/\n/).map((l, i, a) => {
l = l.replace(/</g, "&lt;") // escape html tag opening brackets
.replace(/^({br}|>)$/i, "<pre> </pre>") // add line breaks
// convert headers
.replace(/^### +(.*)/, "<h3 id=\"$1\"><a href=\"#$1\">$1</a></h3>")
.replace(/^## +(.*)/, "<h2 id=\"$1\"><a href=\"#$1\">$1</a></h2>")
.replace(/^# +(.*)/, "<h1 id=\"$1\"><a href=\"#$1\">$1</a></h1>")
// convert images
.replace(/^=> +([a-z0-9\-_\/\.\(\),+:@?!&=#~']+)\.(png|jpg) +(.*)/i, '<img src="$1.$2" alt="$3" title="$3">')
.replace(/^=> +([a-z0-9\-_\/\.\(\),+:@?!&=#~']+)\.(png|jpg)/i, '<img src="$1.$2">')
// convert audio
.replace(/^=> +([a-z0-9\-_\/\.\(\),+:@?!&=#~']+)\.mp3 *(.*)/i,
'<audio controls><source src="$1.mp3" type="audio/mpeg">🔈 audio</audio>')
// convert video
.replace(/^=> +([a-z0-9\-_\/\.\(\),+:@?!&=#~']+)\.mp4 *(.*)/i,
'<video controls><source src="$1.mp4" type="video/mp4">📼 video</video>')
// convert links
.replace(/^=> +([a-z0-9\-_\/\.\(\)%,+:@?!&=#~']+) +(.*)/i, '<a href="$1">$2</a>')
.replace(/^=> +([a-z0-9\-_\/\.\(\)%,+:@?!&=#~']+)/i, '<a href="$1">$1</a>')
// convert block quotes
.replace(/^> *(.*)/, "<blockquote>$1</blockquote>")
// convert lists
.replace(/^[-*+] +(.*)/, "<span class=\"ui\">$1</span>")
.replace(/^([0-9a-bA-B\.]+)\. +(.*)/, "<span class=\"oi\" data-i=\"$1\">$2</span>")
l = hooks(l)
// will this line be considered for its spacing?
if (!l.startsWith("<h") && !l.startsWith("<b") && l.replace(/\n/g, "").length > 0) {
// fetch previous/next lines (default to empty string if not able to acquire)
let previousLine = (a[i - 1] || ""); let nextLine = (a[i + 1] || "")
// check if previous/next line is empty or contains a heading or blockquote
let pLineEmpty = previousLine.length == 0 || previousLine.startsWith("#") || previousLine.startsWith(">")
let nLineEmpty = nextLine.length == 0 || nextLine.startsWith("#") || nextLine.startsWith(">")
if (pLineEmpty && nLineEmpty) // 0 & 0
l = "<p>" + l + "</p>" // single lonely paragraph
else if (pLineEmpty && !nLineEmpty) // 0 & 1
l = "<p>" + l + "<br>" // start a paragraph with line breaks
else if (!pLineEmpty && nLineEmpty) // 1 & 0
l = l + "</p>" // end a paragraph with line breaks
else if (!pLineEmpty && !nLineEmpty) // 1 & 1
l += "<br>" // end a line in a paragraph with a line break
}
return l
}) // remove empty sections and join into one
.filter(x => x.length > 0).join("\n") + "\n"
// if file contains a code block and you _just_ aren't at the end of file
if (_c.length > 1 && i != _c.length - 1) output += "<pre>"
})
return output
}