38b5a8ec5d
If we've already skipped the title, no need to go on searching for it again. We'll probably end up missing some subheadings or something
427 lines
12 KiB
JavaScript
427 lines
12 KiB
JavaScript
const r2 = require('r2')
|
|
const path = require('path')
|
|
const fs = require('fs')
|
|
const { once } = require('events')
|
|
const getPost = require('mediumexporter').getPost
|
|
const { createClient } = require('webdav')
|
|
const readline = require('readline')
|
|
const { markdown } = require('markdown')
|
|
const GhostAdminAPI = require('@tryghost/admin-api')
|
|
const Rembrandt = require('rembrandt')
|
|
|
|
const MEDIUM_IMG_CDN = 'https://miro.medium.com/fit/c/'
|
|
|
|
const ghostAdmin = new GhostAdminAPI({
|
|
url: process.env.GHOST_URL,
|
|
version: process.env.GHOST_VERSION,
|
|
key: process.env.GHOST_ADMIN_KEY
|
|
})
|
|
|
|
/**
|
|
* function [fetchFromMedium]
|
|
* @returns [string] status
|
|
*/
|
|
|
|
const fetchFromMedium = async (mediumUrl) => {
|
|
console.info(`Fetching: ${mediumUrl}`);
|
|
output = path.join(process.env.PWD, 'content')
|
|
|
|
// use mediumexporter's getPost function to fetch a Medium post
|
|
const post = await getPost(mediumUrl, {
|
|
returnObject: true,
|
|
output: output,
|
|
}).catch((err) => {
|
|
return {
|
|
error: err,
|
|
}
|
|
})
|
|
|
|
// set output folder path
|
|
// this is based on what mediumexporter chooses as the output folder
|
|
outputFolder = path.join(output, post.slug)
|
|
|
|
console.info(`Saving to: ${outputFolder}`)
|
|
|
|
if (!fs.existsSync(path.join(outputFolder, post.slug))) {
|
|
fs.mkdirSync(outputFolder, { recursive: true })
|
|
}
|
|
|
|
// mediumexporter writes a plain .md file if the post has no media
|
|
// if that is the case, we should create the subfolder manually
|
|
// and copy the data there.
|
|
if (fs.existsSync(path.join(output, post.slug + '.md'))) {
|
|
fs.renameSync(
|
|
path.join(output, post.slug + '.md'),
|
|
path.join(outputFolder, 'index.md')
|
|
)
|
|
}
|
|
|
|
// generate metadata
|
|
const metadata = JSON.stringify({
|
|
title: post.title,
|
|
subtitle: post.subtitle,
|
|
author: post.author || "",
|
|
authors: post.authors || [],
|
|
date: new Date(post.date),
|
|
tags: post.tags,
|
|
url: post.url,
|
|
slug: post.slug,
|
|
images: post.images,
|
|
featuredImage: post.featuredImage,
|
|
})
|
|
|
|
// write metadata to output folder
|
|
fs.writeFileSync(path.join(outputFolder, 'metadata.json'), metadata)
|
|
return post
|
|
};
|
|
|
|
/**
|
|
* function [pushToGhost]
|
|
* @returns [string] status
|
|
*/
|
|
const pushToGhost = async (postSlug) => {
|
|
console.info('Pushing: ' + postSlug);
|
|
|
|
// Decide working path
|
|
postFolder = path.resolve('content/' + postSlug)
|
|
|
|
// Verify file exists
|
|
if (!fs.existsSync(postFolder)) {
|
|
console.error('Could not find post folder! Is it fetched?')
|
|
return false
|
|
}
|
|
|
|
// Decide file
|
|
const postContent = path.join(postFolder, 'index.md')
|
|
const postOutput = path.join(postFolder, 'ghost.md')
|
|
|
|
// Verify post exists
|
|
if (!fs.existsSync(postContent)) {
|
|
console.error("Could not find 'index.md' in " + postSlug + "! Is it fetched?")
|
|
return false
|
|
}
|
|
|
|
// Decide WebDAV upload path
|
|
current_date = new Date()
|
|
|
|
const uploadPath = path.join(
|
|
current_date.getUTCFullYear().toString(),
|
|
(current_date.getUTCMonth() + 1).toString(),
|
|
postSlug
|
|
)
|
|
|
|
// Path where WebDAV files will be placed (eg. https://example.com:2078)
|
|
const davPath = path.join(process.env.WEBDAV_PATH_PREFIX, uploadPath)
|
|
|
|
// Public path to upload those files (eg. https://media.example.com/uploads)
|
|
// We'll do it directly since path.join mangles the protocol
|
|
const uploadedPath = process.env.WEBDAV_UPLOADED_PATH_PREFIX + '/' + uploadPath
|
|
|
|
// load metadata file
|
|
console.debug('Loading metadata')
|
|
|
|
postMetaFile = path.join(postFolder, 'metadata.json')
|
|
let postMeta = await JSON.parse(fs.readFileSync(postMetaFile))
|
|
|
|
// Process lines
|
|
const readInterface = readline.createInterface({
|
|
input: fs.createReadStream(postContent),
|
|
output: process.stdout,
|
|
terminal: false
|
|
})
|
|
|
|
const outStream = fs.createWriteStream(postOutput, { encoding: 'utf-8' })
|
|
|
|
var titleSkipped = false;
|
|
|
|
let reImage = new RegExp('^!\\[(.*)\\]\\((\\S+?)\\)(.*)')
|
|
let reTitle = new RegExp('^#\ .*')
|
|
|
|
|
|
for await (const line of readInterface) {
|
|
// Line to output
|
|
// Default is to make it same as input
|
|
var newLine = line
|
|
|
|
// Skip the header
|
|
if (!titleSkipped && await reTitle.exec(line)) {
|
|
titleSkipped = true
|
|
}
|
|
|
|
// check for images
|
|
var m = await reImage.exec(line)
|
|
if (m) {
|
|
// Get image name
|
|
var imageAlt = m[1]
|
|
var imageName = m[2].replace('*', '')
|
|
var imagePath = path.join(postFolder, 'images', imageName)
|
|
|
|
if (!fs.existsSync(imagePath)) {
|
|
console.warn('Skipping missing image: ' + imageName)
|
|
} else {
|
|
// check for separator image
|
|
var isScissors = await checkScissors(imagePath)
|
|
if (isScissors) {
|
|
newLine = '\n---\n'
|
|
} else {
|
|
// upload pic to server
|
|
console.debug(`Adding to upload queue: ${imageName}`)
|
|
uploadedImages.push(imageName)
|
|
uploadDav(davPath, imagePath)
|
|
|
|
newLine = '![' + imageAlt + '](' + uploadedPath + '/' + imageName + ')'
|
|
}
|
|
}
|
|
}
|
|
|
|
outStream.write(newLine + '\n')
|
|
}
|
|
|
|
postMetaFile = path.join(postFolder, 'metadata.json')
|
|
let postMeta = JSON.parse(fs.readFileSync(postMetaFile))
|
|
|
|
// calculate users
|
|
let users = []
|
|
postMeta.authors.forEach((user) => {
|
|
users.push({slug: user.username})
|
|
})
|
|
|
|
ghostAdmin.posts.add({
|
|
title: postMeta.title,
|
|
custom_excerpt: postMeta.subtitle || null,
|
|
tags: postMeta.tags,
|
|
authors: users,
|
|
html: markdown.toHTML(fs.readFileSync(postOutput, mode='utf-8'))
|
|
}, {source: 'html'})
|
|
.then((res) => {
|
|
// Check if user was added
|
|
if (res.primary_author.id == 1) {
|
|
console.warn(`WARNING: The admin editor, "${res.primary_author.name}", is set as author for this post. If this is incorrect, there was some problem matching usernames. Please check and set it manually.`)
|
|
}
|
|
}
|
|
}
|
|
|
|
// calculate users
|
|
let users = []
|
|
postMeta.authors.forEach((user) => {
|
|
users.push({slug: user.username})
|
|
})
|
|
|
|
// This will happen once all the line reading is finished
|
|
// Uploads will continue in paralell though
|
|
console.debug('Adding to Ghost')
|
|
|
|
ghostAdmin.posts.add({
|
|
title: postMeta.title,
|
|
custom_excerpt: postMeta.subtitle || null,
|
|
tags: postMeta.tags,
|
|
authors: users,
|
|
html: markdown.toHTML(fs.readFileSync(postOutput, mode='utf-8')),
|
|
feature_image: featuredImagePath
|
|
}, {source: 'html'})
|
|
.then((res) => {
|
|
// Check if user was added
|
|
if (res.primary_author.id == 1) {
|
|
console.warn(`WARNING: The admin editor, "${res.primary_author.name}", is set as author for this post. If this is incorrect, there was some problem matching usernames. Please check and set it manually.`)
|
|
}
|
|
console.log('Post conveyed successfully.')
|
|
})
|
|
};
|
|
|
|
/**
|
|
* function [mediumToGhost]
|
|
* @returns [string] status
|
|
*/
|
|
const mediumToGhost = (mediumUrl) => {
|
|
console.info('Copying: ' + mediumUrl);
|
|
};
|
|
|
|
|
|
async function fetchMediumJSON(mediumUrl) {
|
|
console.debug(`Fetching: ${mediumUrl}`)
|
|
const response = await fetch(mediumUrl)
|
|
const text = await response.text()
|
|
const json = await JSON.parse(text.substr(text.indexOf('{')))
|
|
return json;
|
|
}
|
|
|
|
/**
|
|
* function [checkScissors]
|
|
* @returns [boolean] matchStatus
|
|
*/
|
|
|
|
const checkScissors = async (imagePath) => {
|
|
// Decide "separator" image
|
|
// If set, images matching this will be ignored and replaced
|
|
// with a horizontal-rule ("---" in markdown) instead.
|
|
let scissors = process.env.SEPARATOR_IMAGE
|
|
|
|
// if scissors not set, return false
|
|
// (it's never a scissors since it never matches)
|
|
if (!scissors) {
|
|
console.warn('[scissors] No scissors set, so rejecting all images')
|
|
return false
|
|
} else {
|
|
|
|
// Check if given image matches the scissors
|
|
try {
|
|
let isScissors = new Rembrandt({
|
|
imageA: scissors,
|
|
imageB: imagePath,
|
|
thresholdType: Rembrandt.THRESHOLD_PERCENT,
|
|
maxThreshold: 0.1
|
|
})
|
|
let result = await isScissors.compare()
|
|
return result.passed
|
|
} catch (err) {
|
|
console.warn('[scissors] Skipping scissors check:', err.message)
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* function [createUser]
|
|
* @returns [object] ghost data json
|
|
*/
|
|
const generateUserData = async (mediumUsername, email) => {
|
|
console.debug('Creating: @' + mediumUsername + '(email: ' + email + ')');
|
|
const mediumUrl = `https://medium.com/@${mediumUsername}/?format=json`;
|
|
const json = await fetchMediumJSON(mediumUrl);
|
|
|
|
if (!json.success) {
|
|
console.error(`Error: ${json.error}`)
|
|
return false
|
|
}
|
|
|
|
console.debug(`Name: ${json.payload.user.name}`)
|
|
console.debug(`Bio: ${json.payload.user.bio}`)
|
|
|
|
// Download and upload image
|
|
|
|
let imageId = json.payload.user.imageId
|
|
console.log(`Image: ${imageId}`)
|
|
|
|
let imagePath = MEDIUM_IMG_CDN + '256/256/' + imageId
|
|
let filetype = imageId.split('.')[imageId.split('.').length - 1]
|
|
let fileName = `${mediumUsername}.${filetype}`
|
|
let filePath = path.join(process.env.PWD, fileName)
|
|
|
|
console.log(`Fetching: ${imagePath}`)
|
|
|
|
const response = await (await r2.get(imagePath).response).buffer()
|
|
await fs.writeFileSync(filePath, response, 'base64')
|
|
|
|
console.log("Uploading to server")
|
|
|
|
await uploadDav(path.join(process.env.WEBDAV_PATH_PREFIX,'avatars'),
|
|
filePath)
|
|
|
|
// Generate Ghost JSON
|
|
|
|
const ghostData = {
|
|
data: {
|
|
users: [
|
|
{
|
|
id: 1,
|
|
slug: json.payload.user.username,
|
|
bio: json.payload.user.bio,
|
|
email: email,
|
|
name: json.payload.user.name,
|
|
profile_image: process.env.WEBDAV_UPLOADED_PATH_PREFIX + '/avatars/' + fileName
|
|
}
|
|
]
|
|
},
|
|
meta: {
|
|
exported_on: new Date,
|
|
version: '2.14.0'
|
|
}
|
|
}
|
|
|
|
return(JSON.stringify(ghostData))
|
|
};
|
|
|
|
const createDirIfNotExist = async (client, folder) => {
|
|
// recursively create subfolders if they don't exist.
|
|
|
|
//safety: don't touch directories outside WEBDAV_PATH_PREFIX
|
|
if (!folder.startsWith(process.env.WEBDAV_PATH_PREFIX)) {
|
|
throw new Error(`Cannot create directories outside ${process.env.WEBDAV_PATH_PREFIX}`)
|
|
}
|
|
|
|
// check the folder
|
|
await client.stat(folder)
|
|
.catch(async (err) => {
|
|
if (err.response.status == 404) {
|
|
|
|
// it's a 404, so we'll create the directory
|
|
console.debug(`Noting missing subdirectory: ${folder}`)
|
|
|
|
// first, create the parent directory (if required)
|
|
await createDirIfNotExist(client, path.dirname(folder))
|
|
|
|
console.debug(`Creating missing subdirectory: ${folder}`)
|
|
// then, create the current directory
|
|
await client.createDirectory(folder)
|
|
.catch(async (err) => {
|
|
if (err.response.status == 405) { // Method Not Allowed
|
|
// Maybe the directory's already been created in the meantime?
|
|
await client.stat(folder)
|
|
.catch((err2) => {
|
|
// Bad guess. Panic (and raise the original error)
|
|
console.debug(err.toJSON())
|
|
throw err
|
|
})
|
|
} else {
|
|
// what's this? Panic!
|
|
console.debug(err.toJSON())
|
|
throw err
|
|
}
|
|
})
|
|
} else {
|
|
|
|
// it's not a 404; we don't know how to handle this. Panic!
|
|
console.debug(err.toJSON())
|
|
throw err
|
|
}
|
|
})
|
|
}
|
|
|
|
/**
|
|
* function [createUser]
|
|
* @returns [string] status
|
|
*/
|
|
const uploadDav = async (dirPath, filePath) => {
|
|
|
|
// connect to webdav
|
|
client = createClient(
|
|
process.env.WEBDAV_SERVER_URL,
|
|
{
|
|
username: process.env.WEBDAV_USERNAME,
|
|
password: process.env.WEBDAV_PASSWORD,
|
|
digest: process.env.WEBDAV_USE_DIGEST == 'true'
|
|
})
|
|
|
|
// create directory if not exists
|
|
console.debug(`Loading ${dirPath}`)
|
|
await createDirIfNotExist(client, dirPath)
|
|
|
|
// upload a file
|
|
console.debug('Uploading file')
|
|
s = fs.createReadStream(filePath)
|
|
.pipe(client.createWriteStream(
|
|
path.join(dirPath, path.basename(filePath))
|
|
))
|
|
.on('finish', () => console.debug('Uploaded successfully.'))
|
|
|
|
return true
|
|
}
|
|
|
|
module.exports = {
|
|
fetchFromMedium,
|
|
pushToGhost,
|
|
mediumToGhost,
|
|
generateUserData,
|
|
checkScissors,
|
|
uploadDav
|
|
}
|