pidgin/nest

Refactor and document `pre-prod` and `migrate-tracker`

2019-05-28, Jason
68c614a6a1e1
Parents 127beb82d1ed
Children f1faf9a5483a
Refactor and document `pre-prod` and `migrate-tracker`
2 files changed, 227 insertions(+), 113 deletions(-)
  • +135 -67
    tools/migrate-tracker.js
  • +92 -46
    tools/pre-prod.js
    • --- a/tools/migrate-tracker.js Tue May 28 03:31:05 2019 +0100
    +++ b/tools/migrate-tracker.js Tue May 28 03:56:55 2019 +0100
    @@ -1,81 +1,149 @@
    -const fs = require("fs");
    -const path = require("path");
    -const front = require("front-matter");
    +/**
    + * Tracks the progress of migrating content
    + * from the the old site to the new site
    + */
    +
    +/*****************************************************************************
    + * Imports
    + *****************************************************************************/
    +
    +const fs = require('fs')
    +const path = require('path')
    +const front = require('front-matter')
    +
    +/*****************************************************************************
    + * Set Up
    + *****************************************************************************/
    +
    +const filepath = __dirname.replace(/\/tools(\/)?$/, '/hugo/content')
    +const obsoleteRegex = /\s#(\s+)?obsolete(\s+)?$/i
    +const mdRegex = /\.md$/
    +const migrating = []
    +const obsoleted = []
    +
    +/*****************************************************************************
    + * Execution
    + *****************************************************************************/
    const paths = fs
    - .readFileSync(path.join(__dirname, "paths.txt"), "utf8")
    - .split("\r\n");
    -const obsoleteRegex = /\s#(\s+)?obsolete(\s+)?$/i;
    -const migrating = paths.filter(path => !obsoleteRegex.test("# obsolete"));
    -
    -const covered = new Set();
    -
    -function getFront(path) {
    - try {
    - const data = fs.readFileSync(path, "utf8");
    -
    - const { attributes } = front(data);
    -
    - return attributes;
    - } catch (error) {
    - console.log(error);
    + .readFileSync(path.join(__dirname, 'paths.txt'), 'utf8')
    + .split('\r\n')
    +// Sift each URL
    +paths.forEach(path => {
    + if (obsoleteRegex.test(path)) {
    + obsoleted.push(path.replace(obsoleteRegex, ''))
    + } else {
    + migrating.push(path)
    }
    -}
    -
    -const mdRegex = /\.md$/;
    -
    -function getMdPaths(input) {
    - let output = [];
    - let items = fs.readdirSync(input).map(i => path.join(input, i));
    -
    - while (items.length) {
    - const item = items.pop();
    - const stat = fs.statSync(item);
    -
    - if (stat.isDirectory()) {
    - const nueveau = fs.readdirSync(item).map(i => path.join(item, i));
    - items.push(...nueveau);
    - } else if (stat.isFile() && mdRegex.test(item)) {
    - output.push(item);
    - }
    - }
    -
    - return output.sort();
    -}
    -
    -const filepath = (__dirname.replace(/\/tools(\/)?$/, "/hugo/content"))
    -getMdPaths(path.join(__dirname, "../hugo/content/"))
    +})
    +/** Information on new content */
    +const nestPages = getMdPaths(path.join(__dirname, '../hugo/content/'))
    + // discard translated content
    + .filter(path => !/\.\w+\.md$/.test(path))
    .map(path => {
    - let replaces = getFront(path).replaces;
    -
    - if (!replaces) {
    - replaces = [];
    - } else if (typeof replaces === "string") {
    - replaces = [replaces];
    - } else if (!Array.isArray(replaces)) {
    - throw new Error(
    - `expected replaces of ${path} to be falsey, string or an array`
    - );
    - }
    + const frontMatter = getFrontMatter(path)
    + const replaces = castReplacesDataToArray(frontMatter.replaces)
    return {
    replaces,
    path,
    - relative: path
    - .replace(filepath, "")
    - .replace(/\.md$/, "")
    - .replace(/_index(\.\w\w)?$/, "")
    - };
    + pathOnSite: path
    + // convert to realative file path
    + .replace(filepath, '')
    + // remove file extention
    + .replace(/\.md$/, '')
    + // remove _index
    + .replace(/\/_index(\.\w\w)?$/, ''),
    + }
    })
    - .forEach(page => page.replaces.forEach(path => covered.add(path)));
    +
    +/*****************************************************************************
    + * Generate Stats
    + *****************************************************************************/
    +
    +const covered = new Set()
    +nestPages.forEach(page => page.replaces.forEach(path => covered.add(path)))
    const countOfCovered = migrating.reduce(
    (acc, path) => acc + (covered.has(path) ? 1 : 0),
    0
    -);
    +)
    +
    +const percentComplete = ((countOfCovered / migrating.length) * 100).toPrecision(
    + 2
    +)
    +const countOfObsolete = paths.length - migrating.length
    +
    +console.log(`
    +STATS:
    + tracking ${paths.length} pages
    + ${countOfCovered}/${migrating.length} (${percentComplete}%) migrated
    + ${countOfObsolete} obsoleted
    +`)
    +
    +/*****************************************************************************
    + * Generate redirection mapping
    + *****************************************************************************/
    +
    +// TODO: Generate redirection instructions
    +
    +/*****************************************************************************
    + * Helpers
    + *****************************************************************************/
    +
    +/**
    + * Retrieves front matter from markdown files
    + * @param {string} path path to mardown file
    + */
    +function getFrontMatter(path) {
    + try {
    + const data = fs.readFileSync(path, 'utf8')
    +
    + const { attributes } = front(data)
    +
    + return attributes
    + } catch (error) {
    + console.log(error)
    + }
    +}
    -const percentComplete = (countOfCovered / migrating.length * 100).toPrecision(2);
    -const countOfObsolete = paths.length - migrating.length;
    -console.log(
    - `tracking ${paths.length} pages, ${countOfCovered}/${migrating.length} (${percentComplete}%) migrated, ${countOfObsolete} obsoleted`
    -);
    +/**
    + * creates a list of all markdown files in a directory
    + * @param {string} directory directory to search for markdown files
    + */
    +function getMdPaths(directory) {
    + let output = []
    + let items = fs.readdirSync(directory).map(i => path.join(directory, i))
    +
    + while (items.length) {
    + const item = items.pop()
    + const stat = fs.statSync(item)
    +
    + if (stat.isDirectory()) {
    + const nueveau = fs.readdirSync(item).map(i => path.join(item, i))
    + items.push(...nueveau)
    + } else if (stat.isFile() && mdRegex.test(item)) {
    + output.push(item)
    + }
    + }
    +
    + return output.sort()
    +}
    +
    +/**
    + *
    + * @param {void|string|string[]} replaces
    + */
    +function castReplacesDataToArray(replaces) {
    + if (!replaces) {
    + return []
    + } else if (typeof replaces === 'string') {
    + return [replaces]
    + } else if (Array.isArray(replaces)) {
    + return replaces
    + }
    +
    + throw new Error(
    + `expected replaces of ${path} to be falsey, string or an array`
    + )
    +}
    --- a/tools/pre-prod.js Tue May 28 03:31:05 2019 +0100
    +++ b/tools/pre-prod.js Tue May 28 03:56:55 2019 +0100
    @@ -1,8 +1,21 @@
    -const { html: beautify_html } = require('js-beautify')
    +/**
    + * Runs after Hugo has built the site
    + * * Cleans, Minifies and Optimises hugo output
    + */
    +
    +/*****************************************************************************
    + * Imports
    + *****************************************************************************/
    +
    const fs = require('fs')
    const path = require('path')
    const SVGO = require('svgo')
    +const { html: beautify_html } = require('js-beautify')
    +
    +/*****************************************************************************
    + * Set Up
    + *****************************************************************************/
    const target = path.join(__dirname, '../public/')
    const svgo = new SVGO()
    @@ -11,63 +24,96 @@
    const jsonRegex = /\.json$/
    const svgRegex = /\.svg$/
    -getFilePaths(target, tidyPath)
    +const htmlBeautifyOptions = { max_preserve_newlines: 0 }
    +
    +/*****************************************************************************
    + * Execution
    + *****************************************************************************/
    +
    +getFilePaths(target).forEach(path => {
    + if (path.match(xmlRegex)) {
    + tidyXML(path)
    + } else if (path.match(jsonRegex)) {
    + minifyJSON(path)
    + } else if (path.match(svgRegex)) {
    + optimiseSVG(path)
    + }
    +})
    +
    +/*****************************************************************************
    + * File Handlers
    + *****************************************************************************/
    +
    +/**
    + * Format XML and HTML, removing hugo template artefacts,
    + * like whitespace chasms
    + * @param {string} path
    + */
    +function tidyXML(path) {
    + const data = fs.readFileSync(path, 'utf8')
    + const formatted = beautify_html(data, htmlBeautifyOptions)
    +
    + fs.writeFile(path, formatted, err => {
    + if (err) {
    + console.log(`Error Tidiyng ${path}`, err)
    + }
    + console.log(`Tidied path: ${path.replace(target, '')}`)
    + })
    +}
    -function getFilePaths(input, cb) {
    - let items = readDir(input)
    +/**
    + * Minify JSON
    + * @param {string} path
    + */
    +function minifyJSON(path) {
    + const data = fs.readFileSync(path, 'utf8')
    + const formatted = JSON.stringify(JSON.parse(data))
    +
    + fs.writeFile(path, formatted, err => {
    + if (err) {
    + console.log(`Error minifying ${path}`, err)
    + }
    + console.log(`Minified path: ${path.replace(target, '')}`)
    + })
    +}
    +
    +/**
    + * Optimise SVG files
    + * @param {string} path
    + */
    +function optimiseSVG(path) {
    + const data = fs.readFileSync(path, 'utf8')
    +
    + svgo.optimize(data)
    + .then(d => {
    + fs.writeFileSync(path, d.data)
    + console.log(`Optimised: ${path.replace(target, '')}`)
    + })
    + .catch(err => console.log(`Error Optimising SVG ${path}`, err))
    +}
    +
    +/*****************************************************************************
    + * Helpers
    + *****************************************************************************/
    +
    +function getFilePaths(input) {
    + const items = readDir(input)
    + const ret = []
    while (items.length) {
    const item = items.pop()
    const stat = fs.statSync(item)
    if (stat.isDirectory()) {
    - const nueveau = readDir(item)
    - items.push(...nueveau)
    + items.push(...readDir(item))
    } else if (stat.isFile()) {
    - cb(item)
    + ret.push(item)
    }
    }
    +
    + return ret
    }
    function readDir(item) {
    return fs.readdirSync(item).map(i => path.join(item, i))
    }
    -
    -function tidyPath(path) {
    - if (path.match(xmlRegex)) {
    - const data = fs.readFileSync(path, 'utf8')
    -
    - fs.writeFile(
    - path,
    - beautify_html(data, {
    - max_preserve_newlines: 0,
    - }),
    - err => {
    - if (err) {
    - console.log(`Error Tidiyng ${path}`, err)
    - }
    -
    - console.log(`Tidied path: ${path.replace(target, '')}`)
    - }
    - )
    - } else if (path.match(jsonRegex)) {
    - const data = fs.readFileSync(path, 'utf8')
    -
    - fs.writeFile(path, JSON.stringify(JSON.parse(data)), err => {
    - if (err) {
    - console.log(`Error minifying ${path}`, err)
    - }
    -
    - console.log(`Minified path: ${path.replace(target, '')}`)
    - })
    - } else if (path.match(svgRegex)) {
    - const data = fs.readFileSync(path, 'utf8')
    -
    - svgo.optimize(data)
    - .then(d => {
    - fs.writeFileSync(path, d.data)
    - console.log(`Optimised: ${path.replace(target, '')}`)
    - })
    - .catch(err => console.log(`Error Optimising SVG ${path}`, err))
    - }
    -}