const r2 = require('r2') const path = require('path') const fs = require('fs') const getPost = require('mediumexporter').getPost const { createClient } = require('webdav') const readline = require('readline') const { markdown } = require('markdown') const GhostAdminAPI = require('@tryghost/admin-api') const MEDIUM_IMG_CDN = 'https://miro.medium.com/fit/c/' const ghostAdmin = new GhostAdminAPI({ url: process.env.GHOST_URL, version: process.env.GHOST_VERSION, key: process.env.GHOST_ADMIN_KEY }) /** * function [fetchFromMedium] * @returns [string] status */ const fetchFromMedium = async (mediumUrl) => { console.info(`Fetching: ${mediumUrl}`); output = path.join(process.env.PWD, 'content') // use mediumexporter's getPost function to fetch a Medium post const post = await getPost(mediumUrl, { returnObject: true, output: output, }).catch((err) => { return { error: err, } }) // set output folder path // this is based on what mediumexporter chooses as the output folder outputFolder = path.join(output, post.slug) console.info(`Saving to: ${outputFolder}`) if (!fs.existsSync(path.join(outputFolder, post.slug))) { fs.mkdirSync(outputFolder, { recursive: true }) } // mediumexporter writes a plain .md file if the post has no media // if that is the case, we should create the subfolder manually // and copy the data there. if (fs.existsSync(path.join(output, post.slug + '.md'))) { fs.renameSync( path.join(output, post.slug + '.md'), path.join(outputFolder, 'index.md') ) } // generate metadata const metadata = JSON.stringify({ title: post.title, subtitle: post.subtitle, author: post.author || "", authors: post.authors || [], date: new Date(post.date), tags: post.tags, url: post.url, slug: post.slug, images: post.images, featuredImage: post.featuredImage, }) // write metadata to output folder fs.writeFileSync(path.join(outputFolder, 'metadata.json'), metadata) return post }; /** * function [pushToGhost] * @returns [string] status */ const pushToGhost = (postSlug) => { console.info('Pushing: ' + postSlug); // Decide working path postFolder = path.resolve('content/' + postSlug) // Verify file exists if (!fs.existsSync(postFolder)) { console.error('Could not find post folder! Is it fetched?') return false } // Decide file const postContent = path.join(postFolder, 'index.md') const postOutput = path.join(postFolder, 'ghost.md') // Verify post exists if (!fs.existsSync(postContent)) { console.error("Could not find 'index.md' in " + postSlug + "! Is it fetched?") return false } // Decide WebDAV upload path current_date = new Date() var uploadPath = path.join( current_date.getUTCFullYear().toString(), current_date.getUTCMonth().toString(), postSlug ) // Path where WebDAV files will be placed (eg. https://example.com:2078) var davPath = path.join(process.env.WEBDAV_PATH_PREFIX, uploadPath) // Public path to upload those files (eg. https://media.example.com/uploads) // We'll do it directly since path.join mangles the protocol var uploadedPath = process.env.WEBDAV_UPLOADED_PATH_PREFIX + '/' + uploadPath // Process lines const readInterface = readline.createInterface({ input: fs.createReadStream(postContent), output: process.stdout, terminal: false }) const outStream = fs.createWriteStream(postOutput, { encoding: 'utf-8' }) let titleSkipped = false; let reImage = new RegExp('^!\\[(.*)\\]\\((\\S+?)\\)(.*)') let reTitle = new RegExp('^#\ .*') readInterface.on('line', (line) => { console.log('Line: ' + line + '\n') // Line to output // Default is to make it same as input var newLine = line // Skip the header if (!titleSkipped && reTitle.exec(line)) return // check for images var m = reImage.exec(line) if (m) { // Get image name var imageAlt = m[1] var imageName = m[2].replace('*', '') var imagePath = path.join(postFolder, 'images', imageName) if (!fs.existsSync(imagePath)) { console.warn('Skipping missing image: ' + imageName) } else { // TODO: upload pic /* uploadDav(davPath, imagePath) */ newLine = '![' + imageAlt + '](' + uploadedPath + '/' + imageName + ')' } } outStream.write(newLine + '\n') }).on('close', () => { console.log('Processing over. Pushing to Ghost') // load metadata file postMetaFile = path.join(postFolder, 'metadata.json') let postMeta = JSON.parse(fs.readFileSync(postMetaFile)) // calculate users let users = [] postMeta.authors.forEach((user) => { users.push({slug: user.username}) }) ghostAdmin.posts.add({ title: postMeta.title, custom_excerpt: postMeta.subtitle || null, tags: postMeta.tags, authors: users, html: markdown.toHTML(fs.readFileSync(postOutput, mode='utf-8')) }, {source: 'html'}) .then((res) => { // Check if user was added if (res.primary_author.id == 1) { console.warn(`WARNING: The admin editor, "${res.primary_author.name}", is set as author for this post. If this is incorrect, there was some problem matching usernames. Please check and set it manually.`) } console.log('Post conveyed successfully.') }) }) }; /** * function [mediumToGhost] * @returns [string] status */ const mediumToGhost = (mediumUrl) => { console.info('Copying: ' + mediumUrl); }; async function fetchMediumJSON(mediumUrl) { console.debug(`Fetching: ${mediumUrl}`) const response = await fetch(mediumUrl) const text = await response.text() const json = await JSON.parse(text.substr(text.indexOf('{'))) return json; } /** * function [createUser] * @returns [object] ghost data json */ const generateUserData = async (mediumUsername, email) => { console.debug('Creating: @' + mediumUsername + '(email: ' + email + ')'); const mediumUrl = `https://medium.com/@${mediumUsername}/?format=json`; const json = await fetchMediumJSON(mediumUrl); if (!json.success) { console.error(`Error: ${json.error}`) return false } console.debug(`Name: ${json.payload.user.name}`) console.debug(`Bio: ${json.payload.user.bio}`) // Download and upload image let imageId = json.payload.user.imageId console.log(`Image: ${imageId}`) let imagePath = MEDIUM_IMG_CDN + '256/256/' + imageId let filetype = imageId.split('.')[imageId.split('.').length - 1] let fileName = `${mediumUsername}.${filetype}` let filePath = path.join(process.env.PWD, fileName) console.log(`Fetching: ${imagePath}`) const response = await (await r2.get(imagePath).response).buffer() await fs.writeFileSync(filePath, response, 'base64') console.log("Uploading to server") await uploadDav(path.join(process.env.WEBDAV_PATH_PREFIX,'avatars'), filePath) // Generate Ghost JSON const ghostData = { data: { users: [ { id: 1, slug: json.payload.user.username, bio: json.payload.user.bio, email: email, name: json.payload.user.name, profile_image: process.env.WEBDAV_UPLOADED_PATH_PREFIX + '/avatars/' + fileName } ] }, meta: { exported_on: new Date, version: '2.14.0' } } return(JSON.stringify(ghostData)) }; const createDirIfNotExist = async (client, folder) => { // recursively create subfolders if they don't exist. //safety: don't touch directories outside WEBDAV_PATH_PREFIX if (!folder.startsWith(process.env.WEBDAV_PATH_PREFIX)) { throw new Error(`Cannot create directories outside ${process.env.WEBDAV_PATH_PREFIX}`) } // check the folder await client.stat(folder) .catch(async (err) => { if (err.response.status == 404) { // it's a 404, so we'll create the directory console.debug(`Noting missing subdirectory: ${folder}`) // first, create the parent directory (if required) await createDirIfNotExist(client, path.dirname(folder)) console.debug(`Creating missing subdirectory: ${folder}`) // then, create the current directory await client.createDirectory(folder) .catch(async (err) => { if (err.response.status == 405) { // Method Not Allowed // Maybe the directory's already been created in the meantime? await client.stat(folder) .catch((err2) => { // Bad guess. Panic (and raise the original error) console.debug(err.toJSON()) throw err }) } else { // what's this? Panic! console.debug(err.toJSON()) throw err } }) } else { // it's not a 404; we don't know how to handle this. Panic! console.debug(err.toJSON()) throw err } }) } /** * function [createUser] * @returns [string] status */ const uploadDav = async (dirPath, filePath) => { // connect to webdav client = createClient( process.env.WEBDAV_SERVER_URL, { username: process.env.WEBDAV_USERNAME, password: process.env.WEBDAV_PASSWORD, digest: process.env.WEBDAV_USE_DIGEST == 'true' }) // create directory if not exists console.debug(`Loading ${dirPath}`) await createDirIfNotExist(client, dirPath) // upload a file console.debug('Uploading file') s = fs.createReadStream(filePath) .pipe(client.createWriteStream( path.join(dirPath, path.basename(filePath)) )) .on('finish', () => console.debug('Uploaded successfully.')) return true } module.exports = { fetchFromMedium, pushToGhost, mediumToGhost, generateUserData, uploadDav }