const r2 = require('r2') const path = require('path') const fs = require('fs') const { once } = require('events') const getPost = require('mediumexporter').getPost const { createClient } = require('webdav') const readline = require('readline') const { markdown } = require('markdown') const GhostAdminAPI = require('@tryghost/admin-api') const Rembrandt = require('rembrandt') const config = require('./config') const MEDIUM_IMG_CDN = 'https://miro.medium.com/fit/c/' try { const ghostAdmin = new GhostAdminAPI({ url: config.ghost.url, version: config.ghost.version, key: config.ghost.admin_key, }) } catch(err) { console.error('Your Ghost isn\'t configured. Please run `seance setup` to fix this!') } /** * function [fetchFromMedium] * @returns [string] status */ const fetchFromMedium = async (mediumUrl) => { console.info(`Fetching: ${mediumUrl}`); output = path.join(process.env.PWD, 'content') // use mediumexporter's getPost function to fetch a Medium post const post = await getPost(mediumUrl, { returnObject: true, output: output, }).catch((err) => { return { error: err, } }) // set output folder path // this is based on what mediumexporter chooses as the output folder outputFolder = path.join(output, post.slug) console.info(`Saving to: ${outputFolder}`) if (!fs.existsSync(path.join(outputFolder, post.slug))) { fs.mkdirSync(outputFolder, { recursive: true }) } // mediumexporter writes a plain .md file if the post has no media // if that is the case, we should create the subfolder manually // and copy the data there. if (fs.existsSync(path.join(output, post.slug + '.md'))) { fs.renameSync( path.join(output, post.slug + '.md'), path.join(outputFolder, 'index.md') ) } // generate metadata const metadata = JSON.stringify({ title: post.title, subtitle: post.subtitle, author: post.author || "", authors: post.authors || [], date: new Date(post.date), tags: post.tags, url: post.url, slug: post.slug, images: post.images, featuredImage: post.featuredImage, }) // write metadata to output folder fs.writeFileSync(path.join(outputFolder, 'metadata.json'), metadata) return post }; /** * function [pushToGhost] * @returns [string] status */ const pushToGhost = async (postSlug) => { console.info('Pushing: ' + postSlug); // Decide working path postFolder = path.resolve('content/' + postSlug) // Verify file exists if (!fs.existsSync(postFolder)) { console.error('Could not find post folder! Is it fetched?') return false } // Decide file const postContent = path.join(postFolder, 'index.md') const postOutput = path.join(postFolder, 'ghost.md') // Verify post exists if (!fs.existsSync(postContent)) { console.error("Could not find 'index.md' in " + postSlug + "! Is it fetched?") return false } // Decide WebDAV upload path current_date = new Date() const uploadPath = path.join( current_date.getUTCFullYear().toString(), (current_date.getUTCMonth() + 1).toString(), postSlug ) // Path where WebDAV files will be placed (eg. https://example.com:2078) const davPath = path.join(config.webdav.path_prefix, uploadPath) // Public path to upload those files (eg. https://media.example.com/uploads) // We'll do it directly since path.join mangles the protocol const uploadedPath = config.webdav.uploaded_path_prefix + '/' + uploadPath // load metadata file console.debug('Loading metadata') postMetaFile = path.join(postFolder, 'metadata.json') let postMeta = await JSON.parse(fs.readFileSync(postMetaFile)) // Process lines const readInterface = readline.createInterface({ input: fs.createReadStream(postContent), output: process.stdout, terminal: false }) const outStream = fs.createWriteStream(postOutput, { encoding: 'utf-8' }) var titleSkipped = false; let reImage = new RegExp('^!\\[(.*)\\]\\((\\S+?)\\)(.*)') let reTitle = new RegExp('^#\ .*') // Note down uploaded images var uploadedImages = [] for await (const line of readInterface) { // Line to output // Default is to make it same as input var newLine = line // Skip the header if (!titleSkipped && await reTitle.exec(line)) { titleSkipped = true } // check for images var m = await reImage.exec(line) if (m) { // Get image name var imageAlt = m[1] var imageName = m[2].replace('*', '') var imagePath = path.join(postFolder, 'images', imageName) if (!fs.existsSync(imagePath)) { console.warn('Skipping missing image: ' + imageName) } else { // check for separator image var isScissors = await checkScissors(imagePath) if (isScissors) { newLine = '\n---\n' } else { // upload pic to server console.debug(`Adding to upload queue: ${imageName}`) uploadedImages.push(imageName) uploadDav(davPath, imagePath) newLine = '![' + imageAlt + '](' + uploadedPath + '/' + imageName + ')' } } } outStream.write(newLine + '\n') } // Upload feature_image, if required var featuredImagePath if (!!postMeta.featuredImage) { var imageName = postMeta.featuredImage.replace('*', '') // if the image is listed in postMeta.images, it would have // already been uploaded if (uploadedImages.indexOf(imageName) != -1) { console.log(`Skipping feature image ${imageName}: already listed for upload`) } else { var imagePath = path.join(postFolder, 'images', imageName) // We can only upload if the file exists! if (!fs.existsSync(imagePath)) { console.warn(`Skipping feature image "${imageName}": file not found`) } else { console.log(`Uploading feature image: ${imageName}`) uploadDav(davPath, imagePath) featuredImagePath = uploadedPath + '/' + imageName } } } // calculate users let users = [] postMeta.authors.forEach((user) => { users.push({slug: user.username}) }) // This will happen once all the line reading is finished // Uploads will continue in paralell though console.debug('Adding to Ghost') ghostAdmin.posts.add({ title: postMeta.title, custom_excerpt: postMeta.subtitle || null, tags: postMeta.tags, authors: users, html: markdown.toHTML(fs.readFileSync(postOutput, mode='utf-8')), feature_image: featuredImagePath }, {source: 'html'}) .then((res) => { // Check if user was added if (res.primary_author.id == 1) { console.warn(`WARNING: The admin editor, "${res.primary_author.name}", is set as author for this post. If this is incorrect, there was some problem matching usernames. Please check and set it manually.`) } console.log('Post conveyed successfully.') }) }; /** * function [mediumToGhost] * @returns [string] status */ const mediumToGhost = (mediumUrl) => { console.info('Copying: ' + mediumUrl); }; async function fetchMediumJSON(mediumUrl) { console.debug(`Fetching: ${mediumUrl}`) const response = await fetch(mediumUrl) const text = await response.text() const json = await JSON.parse(text.substr(text.indexOf('{'))) return json; } /** * function [checkScissors] * @returns [boolean] matchStatus */ const checkScissors = async (imagePath) => { // Decide "separator" image // If set, images matching this will be ignored and replaced // with a horizontal-rule ("---" in markdown) instead. let scissors = config.scissors // if scissors not set, return false // (it's never a scissors since it never matches) if (!scissors) { console.warn('[scissors] No scissors set, so rejecting all images') return false } else { // Check if given image matches the scissors try { let isScissors = new Rembrandt({ imageA: scissors, imageB: imagePath, thresholdType: Rembrandt.THRESHOLD_PERCENT, maxThreshold: 0.1 }) let result = await isScissors.compare() return result.passed } catch (err) { console.warn('[scissors] Skipping scissors check:', err.message) return false } } } /** * function [createUser] * @returns [object] ghost data json */ const generateUserData = async (mediumUsername, email) => { console.debug('Creating: @' + mediumUsername + '(email: ' + email + ')'); const mediumUrl = `https://medium.com/@${mediumUsername}/?format=json`; const json = await fetchMediumJSON(mediumUrl); if (!json.success) { console.error(`Error: ${json.error}`) return false } console.debug(`Name: ${json.payload.user.name}`) console.debug(`Bio: ${json.payload.user.bio}`) // Download and upload image let imageId = json.payload.user.imageId console.log(`Image: ${imageId}`) let imagePath = MEDIUM_IMG_CDN + '256/256/' + imageId let filetype = imageId.split('.')[imageId.split('.').length - 1] let fileName = `${mediumUsername}.${filetype}` let filePath = path.join(process.env.PWD, fileName) console.log(`Fetching: ${imagePath}`) const response = await (await r2.get(imagePath).response).buffer() await fs.writeFileSync(filePath, response, 'base64') console.log("Uploading to server") await uploadDav(path.join(config.webdav.path_prefix,'avatars'), filePath) // Generate Ghost JSON const ghostData = { data: { users: [ { id: 1, slug: json.payload.user.username, bio: json.payload.user.bio, email: email, name: json.payload.user.name, profile_image: config.webdav.uploaded_path_prefix + '/avatars/' + fileName } ] }, meta: { exported_on: new Date, version: '2.14.0' } } return(JSON.stringify(ghostData)) }; const createDirIfNotExist = async (client, folder) => { // recursively create subfolders if they don't exist. //safety: don't touch directories outside WEBDAV_PATH_PREFIX if (!folder.startsWith(config.webdav.path_prefix)) { throw new Error(`Cannot create directories outside ${config.webdav.path_prefix}`) } // check the folder await client.stat(folder) .catch(async (err) => { if (!err.response) { // no response! Maybe a network error or something :P console.error(`[dav-upload:folder] Error creating folder "${folder}"`) console.error(`[dav-upload:folder] ${err.toJSON().message}`) console.error('[dav-upload:folder] Please check your Internet connection and try again') return false } else if (err.response.status == 404) { // it's a 404, so we'll create the directory console.debug(`Noting missing subdirectory: ${folder}`) // first, create the parent directory (if required) if (!await createDirIfNotExist(client, path.dirname(folder))) { // if not created, we fail too :-/ return false } console.debug(`Creating missing subdirectory: ${folder}`) // then, create the current directory await client.createDirectory(folder) .catch(async (err) => { if (err.response.status == 405) { // Method Not Allowed // Maybe the directory's already been created in the meantime? await client.stat(folder) .catch((err2) => { // Bad guess. Panic (and raise the original error) console.error(`Error: ${err.toJSON().message}`) console.error("We're not sure what went wrong. Help!") throw err }) } else { // what's this? Panic! console.error(`Error: ${err.toJSON().message}`) console.error("We're not sure what went wrong. Help!") throw err } }) } else { // it's not a 404; we don't know how to handle this. Panic! console.error(err.toJSON()) throw err } }) return true } /** * function [uploadDav] * @returns [string] status */ const uploadDav = async (dirPath, filePath) => { // connect to webdav client = createClient( config.webdav.server_url, { username: config.webdav.username, password: config.webdav.password, digest: config.webdav.use_digest }) // create directory if not exists console.debug(`[dav-upload] Loading ${dirPath}`) if (!await createDirIfNotExist(client, dirPath)) { console.error(`[dav-upload] Could not upload ${path.basename(filePath)} :(`) return false } // upload a file console.debug('Uploading file') outStream = client.createWriteStream( path.join(dirPath, path.basename(filePath)) ) outStream.on('finish', () => console.debug('Uploaded successfully.')) inStream = fs.createReadStream(filePath) .pipe(outStream) return true } module.exports = { fetchFromMedium, pushToGhost, mediumToGhost, generateUserData, checkScissors, uploadDav }