0d9ecfd8f5
Now, the Seance object is what everyone operates with. That way, we can do fancy stuff like emitting events, etc. if the need so arises (as it will, very soon; trust me).
470 lines
14 KiB
JavaScript
470 lines
14 KiB
JavaScript
const r2 = require('r2')
|
|
const path = require('path')
|
|
const fs = require('fs')
|
|
const { once } = require('events')
|
|
const getPost = require('mediumexporter').getPost
|
|
const { createClient } = require('webdav')
|
|
const readline = require('readline')
|
|
const { markdown } = require('markdown')
|
|
const GhostAdminAPI = require('@tryghost/admin-api')
|
|
const Rembrandt = require('rembrandt')
|
|
|
|
const config = require('./config')
|
|
|
|
class Seance {
|
|
constructor(...args) {
|
|
this.MEDIUM_IMG_CDN = 'https://miro.medium.com/fit/c/'
|
|
|
|
try {
|
|
this.ghostAdmin = new GhostAdminAPI({
|
|
url: config.ghost.url,
|
|
version: config.ghost.version,
|
|
key: config.ghost.admin_key,
|
|
})
|
|
} catch(err) {
|
|
console.error('Your Ghost isn\'t configured. Please run `seance setup` to fix this!')
|
|
}
|
|
|
|
/**
|
|
* function [fetchFromMedium]
|
|
* @returns [string] status
|
|
*/
|
|
}
|
|
|
|
async fetchFromMedium (mediumUrl) {
|
|
console.info(`Fetching: ${mediumUrl}`);
|
|
var output = path.join(process.env.PWD, 'content')
|
|
|
|
var json
|
|
|
|
json = await this.fetchMediumJSON(mediumUrl)
|
|
|
|
// use mediumexporter's getPost function to fetch a Medium post
|
|
const post = await getPost(mediumUrl, {
|
|
returnObject: true,
|
|
output: output,
|
|
postJSON: json
|
|
}).catch((err) => {
|
|
return {
|
|
error: err,
|
|
}
|
|
})
|
|
|
|
// set output folder path
|
|
// this is based on what mediumexporter chooses as the output folder
|
|
var outputFolder = path.join(output, post.slug)
|
|
|
|
console.info(`Saving to: ${outputFolder}`)
|
|
|
|
if (!fs.existsSync(path.join(outputFolder, post.slug))) {
|
|
fs.mkdirSync(outputFolder, { recursive: true })
|
|
}
|
|
|
|
// mediumexporter writes a plain .md file if the post has no media
|
|
// if that is the case, we should create the subfolder manually
|
|
// and copy the data there.
|
|
if (fs.existsSync(path.join(output, post.slug + '.md'))) {
|
|
fs.renameSync(
|
|
path.join(output, post.slug + '.md'),
|
|
path.join(outputFolder, 'index.md')
|
|
)
|
|
}
|
|
|
|
// generate metadata
|
|
const metadata = JSON.stringify({
|
|
title: post.title,
|
|
subtitle: post.subtitle,
|
|
author: post.author || "",
|
|
authors: post.authors || [],
|
|
date: new Date(post.date),
|
|
tags: post.tags,
|
|
url: post.url,
|
|
slug: post.slug,
|
|
images: post.images,
|
|
featuredImage: post.featuredImage,
|
|
})
|
|
|
|
// write metadata to output folder
|
|
fs.writeFileSync(path.join(outputFolder, 'metadata.json'), metadata)
|
|
return post
|
|
};
|
|
|
|
|
|
/**
|
|
* function [pushToGhost]
|
|
* @returns [string] status
|
|
*/
|
|
async pushToGhost (postSlug) {
|
|
console.info('Pushing: ' + postSlug);
|
|
|
|
// Decide working path
|
|
var postFolder = path.resolve('content/' + postSlug)
|
|
|
|
// Verify file exists
|
|
if (!fs.existsSync(postFolder)) {
|
|
console.error('Could not find post folder! Is it fetched?')
|
|
return false
|
|
}
|
|
|
|
// Decide file
|
|
const postContent = path.join(postFolder, 'index.md')
|
|
const postOutput = path.join(postFolder, 'ghost.md')
|
|
|
|
// Verify post exists
|
|
if (!fs.existsSync(postContent)) {
|
|
console.error("Could not find 'index.md' in " + postSlug + "! Is it fetched?")
|
|
return false
|
|
}
|
|
|
|
// Decide WebDAV upload path
|
|
var current_date = new Date()
|
|
|
|
const uploadPath = path.join(
|
|
current_date.getUTCFullYear().toString(),
|
|
(current_date.getUTCMonth() + 1).toString(),
|
|
postSlug
|
|
)
|
|
|
|
// Path where WebDAV files will be placed (eg. https://example.com:2078)
|
|
const davPath = path.join(config.webdav.path_prefix, uploadPath)
|
|
|
|
// Public path to upload those files (eg. https://media.example.com/uploads)
|
|
// We'll do it directly since path.join mangles the protocol
|
|
const uploadedPath = config.webdav.uploaded_path_prefix + '/' + uploadPath
|
|
|
|
// load metadata file
|
|
console.debug('Loading metadata')
|
|
|
|
var postMetaFile = path.join(postFolder, 'metadata.json')
|
|
let postMeta = await JSON.parse(fs.readFileSync(postMetaFile))
|
|
|
|
// Process lines
|
|
const readInterface = readline.createInterface({
|
|
input: fs.createReadStream(postContent),
|
|
output: process.stdout,
|
|
terminal: false
|
|
})
|
|
|
|
const outStream = fs.createWriteStream(postOutput, { encoding: 'utf-8' })
|
|
|
|
var titleSkipped = false;
|
|
|
|
let reImage = new RegExp('^!\\[(.*)\\]\\((\\S+?)\\)(.*)')
|
|
let reTitle = new RegExp('^#\ .*')
|
|
|
|
// Note down uploaded images
|
|
var uploadedImages = []
|
|
|
|
for await (const line of readInterface) {
|
|
// Line to output
|
|
// Default is to make it same as input
|
|
var newLine = line
|
|
|
|
// Skip the header
|
|
if (!titleSkipped && await reTitle.exec(line)) {
|
|
titleSkipped = true
|
|
}
|
|
|
|
// check for images
|
|
var m = await reImage.exec(line)
|
|
if (m) {
|
|
// Get image name
|
|
var imageAlt = m[1]
|
|
var imageName = m[2].replace('*', '')
|
|
var imagePath = path.join(postFolder, 'images', imageName)
|
|
|
|
if (!fs.existsSync(imagePath)) {
|
|
console.warn('Skipping missing image: ' + imageName)
|
|
} else {
|
|
// check for separator image
|
|
var isScissors = await this.checkScissors(imagePath)
|
|
if (isScissors) {
|
|
newLine = '\n---\n'
|
|
} else {
|
|
// upload pic to server
|
|
console.debug(`Adding to upload queue: ${imageName}`)
|
|
uploadedImages.push(imageName)
|
|
this.uploadDav(davPath, imagePath)
|
|
|
|
newLine = ''
|
|
}
|
|
}
|
|
}
|
|
|
|
outStream.write(newLine + '\n')
|
|
}
|
|
|
|
// Upload feature_image, if required
|
|
var featuredImagePath
|
|
if (!!postMeta.featuredImage) {
|
|
var imageName = postMeta.featuredImage.replace('*', '')
|
|
|
|
// if the image is listed in postMeta.images, it would have
|
|
// already been uploaded
|
|
if (uploadedImages.indexOf(imageName) != -1) {
|
|
console.log(`Skipping feature image ${imageName}: already listed for upload`)
|
|
} else {
|
|
var imagePath = path.join(postFolder, 'images', imageName)
|
|
|
|
// We can only upload if the file exists!
|
|
if (!fs.existsSync(imagePath)) {
|
|
console.warn(`Skipping feature image "${imageName}": file not found`)
|
|
} else {
|
|
console.log(`Uploading feature image: ${imageName}`)
|
|
this.uploadDav(davPath, imagePath)
|
|
featuredImagePath = uploadedPath + '/' + imageName
|
|
}
|
|
}
|
|
}
|
|
|
|
// calculate users
|
|
let users = []
|
|
postMeta.authors.forEach((user) => {
|
|
users.push({slug: user.username})
|
|
})
|
|
|
|
// This will happen once all the line reading is finished
|
|
// Uploads will continue in paralell though
|
|
console.debug('Adding to Ghost')
|
|
|
|
this.ghostAdmin.posts.add({
|
|
title: postMeta.title,
|
|
custom_excerpt: postMeta.subtitle || null,
|
|
tags: postMeta.tags,
|
|
authors: users,
|
|
html: markdown.toHTML(fs.readFileSync(postOutput, 'utf-8')),
|
|
feature_image: featuredImagePath
|
|
}, {source: 'html'})
|
|
.then((res) => {
|
|
// Check if user was added
|
|
if (res.primary_author.id == 1) {
|
|
console.warn(`WARNING: The admin editor, "${res.primary_author.name}", is set as author for this post. If this is incorrect, there was some problem matching usernames. Please check and set it manually.`)
|
|
}
|
|
console.log('Post conveyed successfully.')
|
|
})
|
|
};
|
|
|
|
|
|
/**
|
|
* function [mediumToGhost]
|
|
* @returns [string] status
|
|
*/
|
|
mediumToGhost (mediumUrl) {
|
|
console.info('Copying: ' + mediumUrl);
|
|
}
|
|
|
|
|
|
async fetchMediumJSON(mediumUrl) {
|
|
var json
|
|
var text
|
|
|
|
if (mediumUrl.match(/^http/i)) {
|
|
// add ?json attribute
|
|
mediumUrl = mediumUrl.replace(/#.*$/, '')
|
|
mediumUrl= `${mediumUrl}?format=json`
|
|
const response = await fetch(mediumUrl)
|
|
text = await response.text()
|
|
} else {
|
|
throw { error: 'URL must be a Medium URL' }
|
|
}
|
|
|
|
json = await JSON.parse(text.substr(text.indexOf('{')))
|
|
|
|
return json;
|
|
}
|
|
|
|
/**
|
|
* function [checkScissors]
|
|
* @returns [boolean] matchStatus
|
|
*/
|
|
|
|
async checkScissors (imagePath) {
|
|
// Decide "separator" image
|
|
// If set, images matching this will be ignored and replaced
|
|
// with a horizontal-rule ("---" in markdown) instead.
|
|
let scissors = config.scissors
|
|
|
|
// if scissors not set, return false
|
|
// (it's never a scissors since it never matches)
|
|
if (!scissors) {
|
|
console.warn('[scissors] No scissors set, so rejecting all images')
|
|
return false
|
|
} else {
|
|
|
|
// Check if given image matches the scissors
|
|
try {
|
|
let isScissors = new Rembrandt({
|
|
imageA: scissors,
|
|
imageB: imagePath,
|
|
thresholdType: Rembrandt.THRESHOLD_PERCENT,
|
|
maxThreshold: 0.1
|
|
})
|
|
let result = await isScissors.compare()
|
|
return result.passed
|
|
} catch (err) {
|
|
console.warn('[scissors] Skipping scissors check:', err.message)
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* function [createUser]
|
|
* @returns [object] ghost data json
|
|
*/
|
|
async generateUserData (mediumUsername, email) {
|
|
console.debug('Creating: @' + mediumUsername + '(email: ' + email + ')');
|
|
const mediumUrl = `https://medium.com/@${mediumUsername}/?format=json`;
|
|
const json = await fetchMediumJSON(mediumUrl);
|
|
|
|
if (!json.success) {
|
|
console.error(`Error: ${json.error}`)
|
|
return false
|
|
}
|
|
|
|
console.debug(`Name: ${json.payload.user.name}`)
|
|
console.debug(`Bio: ${json.payload.user.bio}`)
|
|
|
|
// Download and upload image
|
|
|
|
let imageId = json.payload.user.imageId
|
|
console.log(`Image: ${imageId}`)
|
|
|
|
let imagePath = MEDIUM_IMG_CDN + '256/256/' + imageId
|
|
let filetype = imageId.split('.')[imageId.split('.').length - 1]
|
|
let fileName = `${mediumUsername}.${filetype}`
|
|
let filePath = path.join(process.env.PWD, fileName)
|
|
|
|
console.log(`Fetching: ${imagePath}`)
|
|
|
|
const response = await (await r2.get(imagePath).response).buffer()
|
|
await fs.writeFileSync(filePath, response, 'base64')
|
|
|
|
console.log("Uploading to server")
|
|
|
|
await this.uploadDav(path.join(config.webdav.path_prefix,'avatars'),
|
|
filePath)
|
|
|
|
// Generate Ghost JSON
|
|
|
|
const ghostData = {
|
|
data: {
|
|
users: [
|
|
{
|
|
id: 1,
|
|
slug: json.payload.user.username,
|
|
bio: json.payload.user.bio,
|
|
email: email,
|
|
name: json.payload.user.name,
|
|
profile_image: config.webdav.uploaded_path_prefix + '/avatars/' + fileName
|
|
}
|
|
]
|
|
},
|
|
meta: {
|
|
exported_on: new Date,
|
|
version: '2.14.0'
|
|
}
|
|
}
|
|
|
|
return(JSON.stringify(ghostData))
|
|
};
|
|
|
|
async createDirIfNotExist (client, folder) {
|
|
// recursively create subfolders if they don't exist.
|
|
|
|
//safety: don't touch directories outside WEBDAV_PATH_PREFIX
|
|
if (!folder.startsWith(config.webdav.path_prefix)) {
|
|
throw new Error(`Cannot create directories outside ${config.webdav.path_prefix}`)
|
|
}
|
|
|
|
// check the folder
|
|
await client.stat(folder)
|
|
.catch(async (err) => {
|
|
if (!err.response) {
|
|
// no response! Maybe a network error or something :P
|
|
console.error(`[dav-upload:folder] Error creating folder "${folder}"`)
|
|
console.error(`[dav-upload:folder] ${err.toJSON().message}`)
|
|
console.error('[dav-upload:folder] Please check your Internet connection and try again')
|
|
return false
|
|
} else if (err.response.status == 404) {
|
|
|
|
// it's a 404, so we'll create the directory
|
|
console.debug(`Noting missing subdirectory: ${folder}`)
|
|
|
|
// first, create the parent directory (if required)
|
|
if (!await this.createDirIfNotExist(client, path.dirname(folder))) {
|
|
// if not created, we fail too :-/
|
|
return false
|
|
}
|
|
|
|
console.debug(`Creating missing subdirectory: ${folder}`)
|
|
// then, create the current directory
|
|
await client.createDirectory(folder)
|
|
.catch(async (err) => {
|
|
if (err.response.status == 405) { // Method Not Allowed
|
|
// Maybe the directory's already been created in the meantime?
|
|
await client.stat(folder)
|
|
.catch((err2) => {
|
|
// Bad guess. Panic (and raise the original error)
|
|
console.error(`Error: ${err.toJSON().message}`)
|
|
console.error("We're not sure what went wrong. Help!")
|
|
throw err
|
|
})
|
|
} else {
|
|
// what's this? Panic!
|
|
console.error(`Error: ${err.toJSON().message}`)
|
|
console.error("We're not sure what went wrong. Help!")
|
|
throw err
|
|
}
|
|
})
|
|
} else {
|
|
|
|
// it's not a 404; we don't know how to handle this. Panic!
|
|
console.error(err.toJSON())
|
|
throw err
|
|
}
|
|
})
|
|
return true
|
|
}
|
|
|
|
/**
|
|
* function [uploadDav]
|
|
* @returns [string] status
|
|
*/
|
|
async uploadDav (dirPath, filePath) {
|
|
|
|
// connect to webdav
|
|
const client = createClient(
|
|
config.webdav.server_url,
|
|
{
|
|
username: config.webdav.username,
|
|
password: config.webdav.password,
|
|
digest: config.webdav.use_digest
|
|
})
|
|
|
|
// create directory if not exists
|
|
console.debug(`[dav-upload] Loading ${dirPath}`)
|
|
if (!await this.createDirIfNotExist(client, dirPath)) {
|
|
console.error(`[dav-upload] Could not upload ${path.basename(filePath)} :(`)
|
|
return false
|
|
}
|
|
|
|
// upload a file
|
|
console.debug('Uploading file')
|
|
const outStream = client.createWriteStream(
|
|
path.join(dirPath, path.basename(filePath))
|
|
)
|
|
outStream.on('finish', () => console.debug('Uploaded successfully.'))
|
|
|
|
const inStream = fs.createReadStream(filePath)
|
|
.pipe(outStream)
|
|
|
|
return true
|
|
}
|
|
|
|
}
|
|
|
|
module.exports = {
|
|
Seance
|
|
}
|