seance/functions.js

const r2 = require('r2')
const path = require('path')
const fs = require('fs')
const { once } = require('events')
const getPost = require('mediumexporter').getPost
const { createClient } = require('webdav')
const readline = require('readline')
const { markdown } = require('markdown')
const GhostAdminAPI = require('@tryghost/admin-api')
const Rembrandt = require('rembrandt')

const config = require('./config')

const MEDIUM_IMG_CDN = 'https://miro.medium.com/fit/c/'

const ghostAdmin = new GhostAdminAPI({
  url: config.ghost.url,
  version: config.ghost.version,
  key: config.ghost.admin_key,
})

/**
 * function [fetchFromMedium]
 * @returns [string] status
*/

const fetchFromMedium = async (mediumUrl) => {
  console.info(`Fetching: ${mediumUrl}`);
  output = path.join(process.env.PWD, 'content')

  // use mediumexporter's getPost function to fetch a Medium post
  const post = await getPost(mediumUrl, {
      returnObject: true,
      output: output,
    }).catch((err) => {
      return {
        error: err,
      }
    })

  // set output folder path
  // this is based on what mediumexporter chooses as the output folder
  outputFolder = path.join(output, post.slug)

  console.info(`Saving to: ${outputFolder}`)

  if (!fs.existsSync(path.join(outputFolder, post.slug))) {
    fs.mkdirSync(outputFolder, { recursive: true })
  }

  // mediumexporter writes a plain .md file if the post has no media
  // if that is the case, we should create the subfolder manually
  // and copy the data there.
  if (fs.existsSync(path.join(output, post.slug + '.md'))) {
    fs.renameSync(
      path.join(output, post.slug + '.md'),
      path.join(outputFolder, 'index.md')
    )
  }

  // generate metadata
  const metadata = JSON.stringify({
    title: post.title,
    subtitle: post.subtitle,
    author: post.author || "",
    authors: post.authors || [],
    date: new Date(post.date),
    tags: post.tags,
    url: post.url,
    slug: post.slug,
    images: post.images,
    featuredImage: post.featuredImage,
  })

  // write metadata to output folder
  fs.writeFileSync(path.join(outputFolder, 'metadata.json'), metadata)
  return post
};

/**
 * function [pushToGhost]
 * @returns [string] status
*/
const pushToGhost = async (postSlug) => {
  console.info('Pushing: ' + postSlug);

  // Decide working path
  postFolder = path.resolve('content/' + postSlug)

  // Verify file exists
  if (!fs.existsSync(postFolder)) {
    console.error('Could not find post folder! Is it fetched?')
    return false
  }

  // Decide file
  const postContent = path.join(postFolder, 'index.md')
  const postOutput = path.join(postFolder, 'ghost.md')

  // Verify post exists
  if (!fs.existsSync(postContent)) {
    console.error("Could not find 'index.md' in " + postSlug + "! Is it fetched?")
    return false
  }

  // Decide WebDAV upload path
  current_date = new Date()

  const uploadPath = path.join(
    current_date.getUTCFullYear().toString(),
    (current_date.getUTCMonth() + 1).toString(),
    postSlug
  )

  // Path where WebDAV files will be placed (eg. https://example.com:2078)
  const davPath = path.join(config.webdav.path_prefix, uploadPath)

  // Public path to upload those files (eg. https://media.example.com/uploads)
  // We'll do it directly since path.join mangles the protocol
  const uploadedPath = config.webdav.uploaded_path_prefix + '/' + uploadPath

  // load metadata file
  console.debug('Loading metadata')

  postMetaFile = path.join(postFolder, 'metadata.json')
  let postMeta = await JSON.parse(fs.readFileSync(postMetaFile))

  // Process lines
  const readInterface = readline.createInterface({
    input: fs.createReadStream(postContent),
    output: process.stdout,
    terminal: false
  })

  const outStream = fs.createWriteStream(postOutput, { encoding: 'utf-8' })

  var titleSkipped = false;

  let reImage = new RegExp('^!\\[(.*)\\]\\((\\S+?)\\)(.*)')
  let reTitle = new RegExp('^#\ .*')

  // Note down uploaded images
  var uploadedImages = []

  for await (const line of readInterface) {
    // Line to output
    // Default is to make it same as input
    var newLine = line

    // Skip the header
    if (!titleSkipped && await reTitle.exec(line)) {
      titleSkipped = true
    }

    // check for images
    var m = await reImage.exec(line)
    if (m) {
      // Get image name
      var imageAlt = m[1]
      var imageName = m[2].replace('*', '')
      var imagePath = path.join(postFolder, 'images', imageName)

      if (!fs.existsSync(imagePath)) {
        console.warn('Skipping missing image: ' + imageName)
      } else {
        // check for separator image
        var isScissors = await checkScissors(imagePath)
        if (isScissors) {
          newLine = '\n---\n'
        } else {
          // upload pic to server
          console.debug(`Adding to upload queue: ${imageName}`)
          uploadedImages.push(imageName)
          uploadDav(davPath, imagePath)

          newLine = '![' + imageAlt + '](' + uploadedPath + '/' + imageName + ')'
        }
      }
    }

    outStream.write(newLine + '\n')
  }

  // Upload feature_image, if required
  var featuredImagePath
  if (!!postMeta.featuredImage) {
    var imageName = postMeta.featuredImage.replace('*', '')

    // if the image is listed in postMeta.images, it would have
    // already been uploaded
    if (uploadedImages.indexOf(imageName) != -1) {
      console.log(`Skipping feature image ${imageName}: already listed for upload`)
    } else {
      var imagePath = path.join(postFolder, 'images', imageName)

      // We can only upload if the file exists!
      if (!fs.existsSync(imagePath)) {
        console.warn(`Skipping feature image "${imageName}": file not found`)
      } else {
        console.log(`Uploading feature image: ${imageName}`)
        uploadDav(davPath, imagePath)
        featuredImagePath = uploadedPath + '/' + imageName
      }
    }
  }

  // calculate users
  let users = []
  postMeta.authors.forEach((user) => {
    users.push({slug: user.username})
  })

  // This will happen once all the line reading is finished
  // Uploads will continue in paralell though
  console.debug('Adding to Ghost')

  ghostAdmin.posts.add({
    title: postMeta.title,
    custom_excerpt: postMeta.subtitle || null,
    tags: postMeta.tags,
    authors: users,
    html: markdown.toHTML(fs.readFileSync(postOutput, mode='utf-8')),
    feature_image: featuredImagePath
  }, {source: 'html'})
  .then((res) => {
    // Check if user was added
    if (res.primary_author.id == 1) {
      console.warn(`WARNING: The admin editor, "${res.primary_author.name}", is set as author for this post. If this is incorrect, there was some problem matching usernames. Please check and set it manually.`)
    }
    console.log('Post conveyed successfully.')
  })
};

/**
 * function [mediumToGhost]
 * @returns [string] status
*/
const mediumToGhost = (mediumUrl) => {
  console.info('Copying: ' + mediumUrl);
};


async function fetchMediumJSON(mediumUrl) {
  console.debug(`Fetching: ${mediumUrl}`)
  const response = await fetch(mediumUrl)
  const text = await response.text()
  const json = await JSON.parse(text.substr(text.indexOf('{')))
  return json;
}

/**
 * function [checkScissors]
 * @returns [boolean] matchStatus
*/

const checkScissors = async (imagePath) => {
  // Decide "separator" image
  // If set, images matching this will be ignored and replaced
  // with a horizontal-rule ("---" in markdown) instead.
  let scissors = config.scissors

    // if scissors not set, return false
  // (it's never a scissors since it never matches)
  if (!scissors) {
    console.warn('[scissors] No scissors set, so rejecting all images')
    return false
  } else {

    // Check if given image matches the scissors
    try {
      let isScissors = new Rembrandt({
        imageA: scissors,
        imageB: imagePath,
        thresholdType: Rembrandt.THRESHOLD_PERCENT,
        maxThreshold: 0.1
      })
      let result = await isScissors.compare()
      return result.passed
    } catch (err) {
      console.warn('[scissors] Skipping scissors check:', err.message)
      return false
    }
  }
}

/**
 * function [createUser]
 * @returns [object] ghost data json
*/
const generateUserData = async (mediumUsername, email) => {
  console.debug('Creating: @' + mediumUsername + '(email: ' + email + ')');
  const mediumUrl = `https://medium.com/@${mediumUsername}/?format=json`;
  const json = await fetchMediumJSON(mediumUrl);

  if (!json.success) {
    console.error(`Error: ${json.error}`)
    return false
  }

  console.debug(`Name: ${json.payload.user.name}`)
  console.debug(`Bio: ${json.payload.user.bio}`)

  // Download and upload image

  let imageId = json.payload.user.imageId
  console.log(`Image: ${imageId}`)

  let imagePath = MEDIUM_IMG_CDN + '256/256/' + imageId
  let filetype = imageId.split('.')[imageId.split('.').length - 1]
  let fileName = `${mediumUsername}.${filetype}`
  let filePath = path.join(process.env.PWD, fileName)

  console.log(`Fetching: ${imagePath}`)

  const response = await (await r2.get(imagePath).response).buffer()
  await fs.writeFileSync(filePath, response, 'base64')

  console.log("Uploading to server")

  await uploadDav(path.join(config.webdav.path_prefix,'avatars'),
    filePath)

  // Generate Ghost JSON

  const ghostData = {
    data: {
      users: [
        {
          id: 1,
          slug: json.payload.user.username,
          bio: json.payload.user.bio,
          email: email,
          name: json.payload.user.name,
          profile_image: config.webdav.uploaded_path_prefix + '/avatars/' + fileName
        }
      ]
    },
    meta: {
      exported_on: new Date,
      version: '2.14.0'
    }
  }

  return(JSON.stringify(ghostData))
};

const createDirIfNotExist = async (client, folder) => {
  // recursively create subfolders if they don't exist.

  //safety: don't touch directories outside WEBDAV_PATH_PREFIX
  if (!folder.startsWith(config.webdav.path_prefix)) {
    throw new Error(`Cannot create directories outside ${config.webdav.path_prefix}`)
  }

  // check the folder
  await client.stat(folder)
  .catch(async (err) => {
    if (!err.response) {
      // no response! Maybe a network error or something :P
      console.error(`[dav-upload:folder] Error creating folder "${folder}"`)
      console.error(`[dav-upload:folder] ${err.toJSON().message}`)
      console.error('[dav-upload:folder] Please check your Internet connection and try again')
      return false
    } else if (err.response.status == 404) {

      // it's a 404, so we'll create the directory
      console.debug(`Noting missing subdirectory: ${folder}`)

      // first, create the parent directory (if required)
      if (!await createDirIfNotExist(client, path.dirname(folder))) {
        // if not created, we fail too :-/
        return false
      }

      console.debug(`Creating missing subdirectory: ${folder}`)
      // then, create the current directory
      await client.createDirectory(folder)
      .catch(async (err) => {
        if (err.response.status == 405) { // Method Not Allowed
          // Maybe the directory's already been created in the meantime?
          await client.stat(folder)
          .catch((err2) => {
            // Bad guess. Panic (and raise the original error)
            console.error(`Error: ${err.toJSON().message}`)
            console.error("We're not sure what went wrong. Help!")
            throw err
          })
        } else {
          // what's this? Panic!
          console.error(`Error: ${err.toJSON().message}`)
          console.error("We're not sure what went wrong. Help!")
          throw err
        }
      })
    } else {

      // it's not a 404; we don't know how to handle this. Panic!
      console.error(err.toJSON())
      throw err
    }
  })
  return true
}

/**
 * function [uploadDav]
 * @returns [string] status
*/
const uploadDav = async (dirPath, filePath) => {

  // connect to webdav
  client = createClient(
    config.webdav.server_url,
    {
      username: config.webdav.username,
      password: config.webdav.password,
      digest: config.webdav.use_digest
    })

  // create directory if not exists
  console.debug(`[dav-upload] Loading ${dirPath}`)
  if (!await createDirIfNotExist(client, dirPath)) {
    console.error(`[dav-upload] Could not upload ${path.basename(filePath)} :(`)
    return false
  }

  // upload a file
  console.debug('Uploading file')
  outStream = client.createWriteStream(
    path.join(dirPath, path.basename(filePath))
  )
  outStream.on('finish', () => console.debug('Uploaded successfully.'))

  inStream = fs.createReadStream(filePath)
  .pipe(outStream)

  return true
}

module.exports = {
  fetchFromMedium,
  pushToGhost,
  mediumToGhost,
  generateUserData,
  checkScissors,
  uploadDav
}