seance/seance.js

804 lines
22 KiB
JavaScript

const r2 = require('r2')
const path = require('path')
const fs = require('fs')
const { once, EventEmitter } = require('events')
const { inherits } = require('util')
const getPost = require('mediumexporter').getPost
const { createClient } = require('webdav')
const readline = require('readline')
const { markdown } = require('markdown')
const GhostAdminAPI = require('@tryghost/admin-api')
const { Canvas, Image } = require('canvas')
const Rembrandt = require('rembrandt')
const config = require('./config')
class Seance {
constructor(...args) {
this.MEDIUM_IMG_CDN = 'https://miro.medium.com/fit/c/'
try {
this.ghostAdmin = new GhostAdminAPI({
url: config.ghost.url,
version: config.ghost.version,
key: config.ghost.admin_key,
})
} catch(err) {
console.error('Your Ghost isn\'t configured. Please run `seance setup` to fix this!')
}
/**
* function [fetchFromMedium]
* @returns [string] status
*/
}
async fetchFromMedium (mediumUrl, options = {
json: null,
}) {
this.emit('update', {
status: 'starting',
message: `Fetching: ${mediumUrl}`,
loglevel: 'info'
})
var output = path.join(process.env.PWD, 'content')
var json
if (!options.json) {
json = await this.fetchMediumJSON(mediumUrl)
} else {
json = options.json
}
// use mediumexporter's getPost function to fetch a Medium post
const post = await getPost(mediumUrl, {
returnObject: true,
output: output,
postJSON: json
}).catch((err) => {
return {
error: err,
}
})
// set output folder path
// this is based on what mediumexporter chooses as the output folder
var outputFolder = path.join(output, post.slug)
this.emit('update', {
status: 'normal',
message: `Saving to: ${outputFolder}`,
loglevel: 'info'
})
if (!fs.existsSync(path.join(outputFolder, post.slug))) {
fs.mkdirSync(outputFolder, { recursive: true })
}
// mediumexporter writes a plain .md file if the post has no media
// if that is the case, we should create the subfolder manually
// and copy the data there.
if (fs.existsSync(path.join(output, post.slug + '.md'))) {
fs.renameSync(
path.join(output, post.slug + '.md'),
path.join(outputFolder, 'index.md')
)
}
// generate metadata
const metadata = JSON.stringify({
title: post.title,
subtitle: post.subtitle,
author: post.author || "",
authors: post.authors || [],
date: new Date(post.date),
tags: post.tags,
url: post.url,
slug: post.slug,
images: post.images,
featuredImage: post.featuredImage,
})
// write metadata to output folder
await fs.promises.writeFile(path.join(outputFolder, 'metadata.json'), metadata)
return post
};
/**
* function [pushToGhost]
* @description
* Pre-processes and uploads the given article to Ghost
*
* @param {Boolean} options.noUpload Skip uploading of images
* @param {Boolean} options.noPush Skip pushing to Ghost; just generate the file
* @param {Boolean} options.dryRun Combination of noUpload and noPush
* @returns [string] status
*/
async pushToGhost (postSlug, options={}) {
this.emit('update', {
status: 'starting',
message: 'Starting upload: ' + postSlug,
loglevel: 'info'
})
if (!!options.dryRun) {
options.noUpload = true
options.noPush = true
}
console.log('noUpload', options.noUpload)
// Decide working path
var postFolder = path.resolve('content/' + postSlug)
// Verify file exists
if (!fs.existsSync(postFolder)) {
this.emit('error', {
message: 'Could not find post folder! Is it fetched?',
})
return false
}
// Decide file
const postContent = path.join(postFolder, 'index.md')
const postOutput = path.join(postFolder, 'ghost.md')
// Verify post exists
if (!fs.existsSync(postContent)) {
this.emit('error', {
message: "Could not find 'index.md' in " + postSlug + "! Is it fetched?",
})
return false
}
// Decide file/WebDAV upload path
var current_date = new Date()
const uploadPath = path.join(
current_date.getUTCFullYear().toString(),
(current_date.getUTCMonth() + 1).toString(),
postSlug
)
// Public path to upload those files (eg. https://example.com:2078)
// We'll do it directly since path.join mangles the protocol
const uploadedPath = config.uploaded_path_prefix + '/' + uploadPath
// load metadata file
this.emit('update', {
status: 'starting',
message: 'Loading metadata',
loglevel: 'debug'
})
var postMetaFile = path.join(postFolder, 'metadata.json')
let postMeta = await JSON.parse(await fs.promises.readFile(postMetaFile))
// Process lines
const readInterface = readline.createInterface({
input: fs.createReadStream(postContent),
output: process.stdout,
terminal: false
})
const outStream = fs.createWriteStream(postOutput, { encoding: 'utf-8' })
// We'll calculate these later since Medium messes it up sometimes
let title = null
let subtitle = null
let reImage = new RegExp('^!\\[(.*)\\]\\((\\S+?)\\)(.*)')
let reTitle = new RegExp('^#\ (.*)')
let reSubtitle = new RegExp('^#+\ (.*)$')
// Note down uploaded images
var uploadedImages = []
this.emit('update', {
status: 'progress',
progress: null, // we don't know the percentage
message: 'Parsing post',
loglevel: 'info'
})
for await (const line of readInterface) {
// Line to output
// Default is to make it same as input
var newLine = line
// Skip the header (and preceding blank lines)
if (!title) {
// blanks
if (!line) continue
// starting with a # (must be the title)
let match = await reTitle.exec(line)
if (match) {
title = match[1]
continue // no need to add line; it'll come automatically
}
} else if (!subtitle) {
// check if it's a repeat of the title (Medium does that)
if (line.endsWith(title)) continue
// otherwise set the subtitle if it doesn't exist
// or if it's a repeat of the title (Medium does that too)
if (!subtitle && postMeta.subtitle == postMeta.title) {
let match = await reSubtitle.exec(line)
if (match) {
subtitle = match[1]
postMeta.subtitle = match[1]
}
}
}
// check for images
var m = await reImage.exec(line)
if (m) {
// Get image name
var imageAlt = m[1]
var imageName = m[2].replace('*', '')
var imagePath = path.join(postFolder, 'images', imageName)
if (!fs.existsSync(imagePath)) {
console.warn('Skipping missing image: ' + imageName)
} else {
// check for separator image
var isScissors = await this.checkScissors(imagePath)
if (isScissors) {
newLine = '\n---\n'
} else {
// upload pic to server
console.debug(`Adding to upload queue: ${imageName}`)
uploadedImages.push(imageName)
// Let's wait for the upload, just to avoid conflicts
if (!options.noUpload) {
await this.upload(uploadPath, imagePath)
}
newLine = '![' + imageAlt + '](' + uploadedPath + '/' + imageName + ')'
}
}
}
outStream.write(newLine + '\n')
}
// Upload feature_image, if required
var featuredImagePath
if (!!postMeta.featuredImage) {
var imageName = postMeta.featuredImage.replace('*', '')
// if the image is listed in postMeta.images, it would have
// already been uploaded
if (uploadedImages.indexOf(imageName) != -1) {
this.emit('update', {
status: 'progress',
progress: 95, // we don't know the percentage
message: `Skipping feature image ${imageName}: already listed for upload`,
loglevel: 'info'
})
} else {
var imagePath = path.join(postFolder, 'images', imageName)
// We can only upload if the file exists!
if (!fs.existsSync(imagePath)) {
this.emit('update', {
status: 'progress',
progress: 95, // we don't know the percentage
message: `Skipping feature image "${imageName}": file not found`,
loglevel: 'warning'
})
} else {
this.emit('update', {
status: 'progress',
progress: 95, // we don't know the percentage
message: `Uploading feature image: ${imageName}`,
loglevel: 'info'
})
if (!options.noUpload) {
this.upload(uploadPath, imagePath)
}
featuredImagePath = uploadedPath + '/' + imageName
}
}
}
// calculate users
let users = []
postMeta.authors.forEach((user) => {
users.push({slug: user.username})
})
// This will happen once all the line reading is finished
// Uploads will continue in paralell though
this.emit('update', {
status: 'progress',
progress: 100, // we don't know the percentage
message: 'Uploading to Ghost',
loglevel: 'info'
})
if (!options.noPush) {
let res = await this.ghostAdmin.posts.add({
title: postMeta.title,
custom_excerpt: postMeta.subtitle || null,
tags: postMeta.tags,
authors: users,
html: markdown.toHTML(await fs.promises.readFile(postOutput, 'utf-8')),
feature_image: featuredImagePath
}, {source: 'html'})
// Check if user was added
if (res.primary_author.id == 1) {
this.emit('notification', {
message: `WARNING: The admin editor, "${res.primary_author.name}", is set as author for this post. If this is incorrect, there was some problem matching usernames. Please check and set it manually.`,
})
}
this.emit('update', {
status: 'progress',
progress: 100, // we don't know the percentage
message: 'Post conveyed successfully',
loglevel: 'info'
})
return {
slug: res.slug,
id: res.id,
uuid: res.uuid,
preview_url: res.url,
primary_author: res.primary_author,
title: res.title,
subtitle: res.custom_excerpt,
status: res.status,
}
} else {
// just return without pushing to Ghost
return {
slug: postSlug,
id: 0,
uuid: 0,
preview_url: null,
primary_author: {},
title: postMeta.title,
subtitle: postMeta.subtitle,
status: 'none',
}
}
}
/**
* function [mediumToGhost]
* @returns [string] status
*/
mediumToGhost (mediumUrl) {
console.info('Copying: ' + mediumUrl);
}
async fetchMediumJSON(mediumUrl) {
var json
var text
if (mediumUrl.match(/^http/i)) {
// remove the anchors at the end
mediumUrl = mediumUrl.replace(/#.*$/, '')
// intelligently add ?json attribute
if (mediumUrl.indexOf('format=json') == -1) {
if (mediumUrl.indexOf('?') == -1) {
mediumUrl = `${mediumUrl}?format=json`
} else {
mediumUrl = `${mediumUrl}&format=json`
}
}
// let's get it!
const response = await fetch(mediumUrl)
text = await response.text()
} else if (fs.existsSync(mediumUrl)) {
text = (await fs.promises.readFile(mediumUrl)).toString()
} else {
throw { error: 'URL must be a Medium URL or existing JSON file' }
}
try {
json = await JSON.parse(text.substr(text.indexOf('{')))
} catch(err) {
throw { error: 'You JSON seems to be malformed' }
}
return json;
}
/**
* function [checkScissors]
* @returns [boolean] matchStatus
*/
async checkScissors (imagePath) {
// Decide "separator" image
// If set, images matching this will be ignored and replaced
// with a horizontal-rule ("---" in markdown) instead.
let scissors = config.scissors
// if scissors not set, return false
// (it's never a scissors since it never matches)
if (!scissors) {
this.emit('update', {
status: 'normal',
message: '[scissors] No scissors set, so rejecting all images',
loglevel: 'warning'
})
return false
} else {
/* First, check that the image has finished loading
* (we don't use this, because Rembrandt loads it again
* on its own, which is messy but what to do ¯\_(ツ)_/¯
*/
try {
let ctx = new Canvas().getContext('2d')
let img = new Image()
img.src = imagePath
ctx.drawImage(img, 0, 0, img.width, img.height)
} catch (err) {
this.emit('update', {
status: 'normal',
message: `[scissors] Skipping scissors check:${err.message}`,
loglevel: 'warning'
})
return false
}
// Check if given image matches the scissors
try {
let isScissors = new Rembrandt({
imageA: scissors,
imageB: imagePath,
thresholdType: Rembrandt.THRESHOLD_PERCENT,
maxThreshold: 0.1
})
let result = await isScissors.compare()
return result.passed
} catch (err) {
this.emit('update', {
status: 'normal',
message: `[scissors] Skipping scissors check:${err.message}`,
loglevel: 'warning'
})
return false
}
}
}
/**
* function [createUser]
* @returns [object] ghost data json
*/
async generateUserData (mediumUsername, email) {
this.emit('update', {
status: 'starting',
message: `Creating: @${mediumUsername} (email: ${email})`,
loglevel: 'debug'
})
const mediumUrl = `https://medium.com/@${mediumUsername}/?format=json`;
const json = await this.fetchMediumJSON(mediumUrl);
if (!json.success) {
this.emit('error', {
message: `Error: ${json.error}`,
})
return false
}
this.emit('update', {
status: 'normal',
message: `Name: ${json.payload.user.name}`,
loglevel: 'debug'
})
this.emit('update', {
status: 'normal',
message: `Bio: ${json.payload.user.bio}`,
loglevel: 'debug'
})
// Download and upload image
let imageId = json.payload.user.imageId
this.emit('update', {
status: 'normal',
message: `Profile pic: ${imageId}`,
loglevel: 'debug'
})
let imagePath = this.MEDIUM_IMG_CDN + '256/256/' + imageId
let filetype = imageId.split('.')[imageId.split('.').length - 1]
let fileName = `${mediumUsername}.${filetype}`
let filePath = path.join(process.env.PWD, fileName)
this.emit('update', {
status: 'normal',
message: `Fetching profile pic: ${imagePath}`,
loglevel: 'info'
})
const response = await (await r2.get(imagePath).response).buffer()
await await fs.promises.writeFile(filePath, response, 'base64')
this.emit('update', {
status: 'normal',
message: `Uploading profile pic: ${imagePath}`,
loglevel: 'info'
})
await this.upload('avatars', filePath)
// Generate Ghost JSON
const ghostData = {
data: {
users: [
{
id: 1,
slug: json.payload.user.username,
bio: json.payload.user.bio,
email: email,
name: json.payload.user.name,
profile_image: config.uploaded_path_prefix + '/avatars/' + fileName
}
]
},
meta: {
exported_on: new Date,
version: '2.14.0'
}
}
return(JSON.stringify(ghostData))
};
async createDirIfNotExist (client, folder) {
// recursively create subfolders if they don't exist.
//safety: don't touch directories outside WEBDAV_PATH_PREFIX
if (!folder.startsWith(config.webdav.path_prefix)) {
throw new Error(`Cannot create directories outside ${config.webdav.path_prefix}`)
}
// check the folder
await client.stat(folder)
.catch(async (err) => {
if (!err.response) {
// no response! Maybe a network error or something :P
console.error(`[dav-upload:folder] Error creating folder "${folder}"`)
console.error(`[dav-upload:folder] ${err.toJSON().message}`)
console.error('[dav-upload:folder] Please check your Internet connection and try again')
return false
} else if (err.response.status == 404) {
// it's a 404, so we'll create the directory
this.emit('update', {
status: 'normal',
message: `Noting missing subdirectory: ${folder}`,
loglevel: 'debug'
})
// first, create the parent directory (if required)
if (!await this.createDirIfNotExist(client, path.dirname(folder))) {
// if not created, we fail too :-/
return false
}
this.emit('update', {
status: 'normal',
message: `Creating missing subdirectory: ${folder}`,
loglevel: 'debug'
})
// then, create the current directory
await client.createDirectory(folder)
.catch(async (err) => {
if (err.response.status == 405) { // Method Not Allowed
// Maybe the directory's already been created in the meantime?
await client.stat(folder)
.catch((err2) => {
// Bad guess. Panic (and raise the original error)
this.emit('update', {
status: 'error',
message: `Error: ${err.toJSON().message}\nWe're not sure what went wrong. Help!`,
loglevel: 'error'
})
throw err
})
} else {
// what's this? Panic!
this.emit('update', {
status: 'error',
message: `Error: ${err.toJSON().message}\nWe're not sure what went wrong. Help!`,
loglevel: 'error'
})
throw err
}
})
} else {
// it's not a 404; we don't know how to handle this. Panic!
this.emit('update', {
status: 'error',
message: 'An unknown error occured. Help!',
loglevel: 'error'
})
console.error(err.toJSON())
throw err
}
})
return true
}
/**
* function [uploadDav]
* @returns [string] status
*/
async uploadDav (dirPath, filePath) {
// Set uploadPath
// We'll do it directly since path.join mangles the protocol
let uploadPath = path.join(config.webdav.path_prefix, dirPath)
// connect to webdav
const client = createClient(
config.webdav.server_url,
{
username: config.webdav.username,
password: config.webdav.password,
digest: config.webdav.use_digest
})
// create directory if not exists
console.debug(`[dav-upload] Loading ${dirPath}`)
if (!await this.createDirIfNotExist(client, uploadPath)) {
console.error(`[dav-upload] Could not upload ${path.basename(filePath)} :(`)
return false
}
// upload a file
console.debug('Uploading file')
const outStream = client.createWriteStream(
path.join(uploadPath, path.basename(filePath))
)
outStream.on('finish', () => console.debug('Uploaded successfully.'))
const inStream = fs.createReadStream(filePath)
.pipe(outStream)
return true
}
/**
* function [exists]
* @returns [boolean]
*
* check if the given file exists or not
*/
async fsExists (path) {
try {
await fs.promises.access(path)
return true
} catch {
return false
}
}
/**
* function [uploadLocal]
* @returns [string] status
*
* upload to a local file path. This should technically be
* called "copy" and not "upload", but the equivalent
* WebDAV one is actually an upload so ¯\_(ツ)_/¯
*/
async uploadLocal (dirPath, filePath) {
// Set uploadPath
// We'll do it directly since path.join mangles the protocol
let uploadPath = path.join(config.local_upload.path_prefix, dirPath)
// safety: don't touch directories outside LOCAL_UPLOAD_PATH_PREFIX
if (!uploadPath.startsWith(config.local_upload.path_prefix)) {
console.error(`[local-upload] Cannot create directories outside ${config.local_upload.path_prefix}`)
return false
}
// create directory if not exists
console.debug(`[local-upload] Loading ${uploadPath}`)
if (
!(await this.fsExists(uploadPath)) ||
!(await fs.promises.lstat(uploadPath)).isDirectory()
) {
if (!(await fs.promises.mkdir(uploadPath,
{ recursive: true }))) {
console.error(`[local-upload] Could not upload ${path.basename(filePath)} :(`)
return false
}
}
// actually do the copying
console.debug('Uploading file')
try {
await fs.promises.copyFile(filePath,
path.join(uploadPath, path.basename(filePath)))
} catch (err) {
console.error(`Upload error: ${err}`)
}
return true
}
/**
* function [upload]
* @returns [boolean] status
*
* upload to WebDAV or a local folder, whichever is configured.
* If both are considered, WebDAV will be preferred.
*/
async upload (dirPath, filePath) {
if (
!!config.webdav &&
!!config.webdav.server_url &&
!!config.webdav.path_prefix
) {
return await this.uploadDav(dirPath, filePath)
} else if (
!!config.local_upload &&
!!config.local_upload.path_prefix
) {
return await this.uploadLocal(dirPath, filePath)
} else {
throw { error: 'Either webdav or local_upload settings must be configured!' }
}
}
}
// Make Seance an EventEmitter
inherits(Seance, EventEmitter)
module.exports = {
Seance
}