From 42f96c00b9a2e936730d9833f9a6051c1d4c5a4c Mon Sep 17 00:00:00 2001 From: Hippo Date: Wed, 21 Jul 2021 22:24:45 +0530 Subject: [PATCH] Allow uploading files locally instead of through WebDAV This gives us more flexibility: after all how often does one have a WebDAV server handy (unless you're smart enough to use HelioHost or some other awesome server)? --- README.md | 19 +++++++--- cli.js | 93 +++++++++++++++++++++++++++++++++------------- config.js | 34 ++++++++++++++++- seance.js | 109 +++++++++++++++++++++++++++++++++++++++++++++++------- 4 files changed, 210 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 984e1e5..5085a22 100644 --- a/README.md +++ b/README.md @@ -40,10 +40,15 @@ files, and for your Ghost API interface. The parameters to set are: * `WEBDAV_SERVER_URL` - location of your WebDAV server * `WEBDAV_USERNAME` - username for signing in * `WEBDAV_PASSWORD` - password, likewise -* `WEBDAV_UPLOADED_PATH` - path where uploaded images will be served (it - could end up being different from `WEBDAV_SERVER_URL`: say you go to - `https://myhost.com:1234/dav/[folder]` to upload, but the public sees - it as `https://media.mysite.com/[folder]`. +* `WEBDAV_PATH_PREFIX` - prefix to add to all WebDAV paths: no uploads + will happen outside of this path +* `UPLOADED_PATH_PREFIX` - path where uploaded images will be + served (it could end up being different from `WEBDAV_SERVER_URL`: say + you go to `https://myhost.com:1234/dav/[folder]` to upload, but the + public sees it as `https://media.mysite.com/[folder]`—or, more + significantly, when you're doing a local-directory upload! +* `LOCAL_UPLOAD_PATH_PREFIX` - path where uploaded images will be copied + locally, if you choose not to use WebDAV * `GHOST_URL` - URL of your Ghost installation * `GHOST_VERSION` - 'v2' or 'v3' depending on which version you're using * `GHOST_ADMIN_KEY` - 'Admi API key for Ghost' @@ -51,7 +56,11 @@ files, and for your Ghost API interface. The parameters to set are: In case you're wondering about the WebDAV server: that's the setup we use at Snipette. We'd like to eventually let you upload directly through Ghost as well, but we're prioritising our setup first to get running -before we think of anything else. Pull requests are welcome! +before we think of anything else. + +Now, we've got a "local upload" option as well which basically copies +the file to a specified directory on the system. Pull requests for +anything else are welcome! ## Pull a post from Medium diff --git a/cli.js b/cli.js index 06c2ee5..f3f53f9 100755 --- a/cli.js +++ b/cli.js @@ -42,39 +42,82 @@ program.command('setup') '\n\nWe\'re going to take you through some steps' + ' to set up your system.\n' ) - console.log('First up: WebDAV details.') - console.log( - 'Please enter your server url (including the port), ' + - 'username, and password\n' - ) var res prompt.start() - res = await prompt.get([ - { name: 'server_url', default: config.webdav.server_url || '' }, - { name: 'username', default: config.webdav.username || '' }, - { name: 'password', default: config.webdav.password || '' , hidden: true}, - ]) - config.webdav.server_url = res.server_url - config.webdav.username = res.username - config.webdav.password = res.password - console.log(`\nOkay. So we have ${config.webdav.username} on ${config.webdav.server_url} with [ the password you set]`) + console.log('First up: File uploads.') console.log( - '\nA couple more settings for your WebDAV: ' + - 'we need to know the path prefix and the uploaded path prefix.\n' + - 'The path prefix is the subfolder to which you upload, like ' + - '`/seance-uploads`, while the uploaded path prefix is what '+ - 'you\'d stick in front of the filename after uploading ' + - '(like `https://media.mysite.com/seance-uploads`).\n' + 'Would you like to upload your files via WebDAV, or just ' + + 'copy them to a local folder on your filesystem? Type ' + + '"webdav" or "local" to choose.\n' ) + res = await prompt.get([ - { name: 'path_prefix', default: config.webdav.path_prefix || '' }, - { name: 'uploaded_path_prefix', default: config.webdav.uploaded_path_prefix || '' }, + { + name: 'upload_mode', + default: 'webdav', + pattern: /^(webdav|local)$/ig, + message: 'Please enter "webdav" or "local"', + }, ]) - config.webdav.path_prefix = res.path_prefix - config.webdav.uploaded_path_prefix = res.uploaded_path_prefix - console.log(`Cool. So uploads to ${config.webdav.path_prefix} will go to ${config.webdav.uploaded_path_prefix}.`) + + if (res.upload_mode == 'webdav') { + + console.log('You\'re going with WebDAV? Awesome!') + console.log( + 'Please enter your server url (including the port), ' + + 'username, and password\n' + ) + + res = await prompt.get([ + { name: 'server_url', default: config.webdav.server_url || '' }, + { name: 'username', default: config.webdav.username || '' }, + { name: 'password', default: config.webdav.password || '' , hidden: true}, + ]) + config.webdav.server_url = res.server_url + config.webdav.username = res.username + config.webdav.password = res.password + console.log(`\nOkay. So we have ${config.webdav.username} on ${config.webdav.server_url} with [ the password you set]`) + + console.log( + '\nA couple more settings for your WebDAV: ' + + 'we need to know the path prefix and the uploaded path prefix.\n' + + 'The path prefix is the subfolder to which you upload, like ' + + '`/seance-uploads`, while the uploaded path prefix is what '+ + 'you\'d stick in front of the filename after uploading ' + + '(like `https://media.mysite.com/seance-uploads`).\n' + ) + res = await prompt.get([ + { name: 'path_prefix', default: config.webdav.path_prefix || '' }, + { + name: 'uploaded_path_prefix', + default: config.uploaded_path_prefix || config.webdav.uploaded_path_prefix || '' + }, + ]) + config.webdav.path_prefix = res.path_prefix + config.uploaded_path_prefix = res.uploaded_path_prefix + console.log(`Cool. So uploads to ${config.webdav.path_prefix} will be visible at ${config.uploaded_path_prefix}.`) + + } else if (res.upload_mode == 'local') { + console.log('You\'re saving files locally? Smart!') + console.log( + 'Two settings we need to know to get things running ' + + 'smoothly: we need the local path/folder where you\'ll be ' + + 'uploading the files, and the uploaded path prefix.\n' + + 'The local path is the folder to which you upload, like ' + + '`/var/www/seance-uploads`, while the uploaded path prefix ' + + 'is what you\'d stick in front of the filename after ' + + 'uploading (like `https://media.mysite.com/seance-uploads`).\n' + ) + res = await prompt.get([ + { name: 'path_prefix', default: config.local_upload.path_prefix || '' }, + { name: 'uploaded_path_prefix', default: config.uploaded_path_prefix || '' }, + ]) + config.local_upload.path_prefix = res.path_prefix + config.uploaded_path_prefix = res.uploaded_path_prefix + console.log(`Cool. So uploads to ${config.local_upload.path_prefix} will be visible at ${config.uploaded_path_prefix}.`) + } console.log('\n\nNext up: Ghost settings.') console.log( diff --git a/config.js b/config.js index c67ca59..295fc31 100644 --- a/config.js +++ b/config.js @@ -11,18 +11,26 @@ convict.addFormats(convict_format_with_validator) convict.addParser({ extension: ['yml', 'yaml'], parse: yaml.safeLoad }) let config = convict({ + uploaded_path_prefix: { + doc: 'URL where files are uploaded (eg. https://mysitem.com/media)', + format: 'url', + env: 'UPLOADED_PATH_PREFIX', + default: null, + }, webdav: { server_url: { doc: 'WebDAV server URL (eg. https://myhost.com:2078)', format: 'url', env: 'WEBDAV_SERVER_URL', default: null, + nullable: true, }, username: { doc: 'Username for WebDAV server', format: 'String', env: 'WEBDAV_USERNAME', default: null, + nullable: true, }, password: { doc: 'Password for WebDAV server', @@ -30,18 +38,21 @@ let config = convict({ env: 'WEBDAV_PASSWORD', default: null, sensitive: true, + nullable: true, }, path_prefix: { doc: 'Where to upload files (eg. /seance-uploads)', format: 'String', env: 'WEBDAV_PATH_PREFIX', default: null, + nullable: true, }, - uploaded_path_prefix: { + uploaded_path_prefix: { // FIXME: Deprecated; remove doc: 'URL where files are uploaded (eg. https://mysitem.com/media)', format: 'url', env: 'WEBDAV_UPLOADED_PATH_PREFIX', default: null, + nullable: true, }, use_digest: { doc: 'Whether to use digest authentication', @@ -50,6 +61,14 @@ let config = convict({ default: false, } }, + local_upload: { + path_prefix: { + doc: 'Where to upload files locally (eg. /media/seance-uploads)', + format: 'String', + env: 'LOCAL_UPLOAD_PATH_PREFIX', + default: null, + }, + }, ghost: { url: { doc: 'URL of Ghost installation', @@ -75,7 +94,7 @@ let config = convict({ format: '*', // TODO: validate by checking path env: 'SEPARATOR_IMAGE', default: null, - } + }, }) // Load configs from home directory, if present @@ -100,6 +119,17 @@ try { validated = false } +// Update deprecated value: config.webdav.uploaded_path_prefix +if (!!config.webdav && !!config.webdav.uploaded_path_prefix) { + console.warn( + 'Warning: config.webdav.uploaded_path_prefix and the ' + + 'WEBDAV_UPLOADED_PATH_PREFIX environment variable are ' + + 'deprecated! Please use config.uploaded_path_prefix or ' + + 'the UPLOADED_PATH_PREFIX environment variable instead.' + ) + config.uploaded_path_prefix = config.webdav.uploaded_path_prefix +} + allConf = config.getProperties() allConf.validated = validated diff --git a/seance.js b/seance.js index 0161ee9..b924780 100644 --- a/seance.js +++ b/seance.js @@ -159,7 +159,7 @@ class Seance { return false } - // Decide WebDAV upload path + // Decide file/WebDAV upload path var current_date = new Date() const uploadPath = path.join( @@ -168,12 +168,9 @@ class Seance { postSlug ) - // Path where WebDAV files will be placed (eg. https://example.com:2078) - const davPath = path.join(config.webdav.path_prefix, uploadPath) - - // Public path to upload those files (eg. https://media.example.com/uploads) + // Public path to upload those files (eg. https://example.com:2078) // We'll do it directly since path.join mangles the protocol - const uploadedPath = config.webdav.uploaded_path_prefix + '/' + uploadPath + const uploadedPath = config.uploaded_path_prefix + '/' + uploadPath // load metadata file this.emit('update', { @@ -270,7 +267,7 @@ class Seance { // Let's wait for the upload, just to avoid conflicts if (!options.noUpload) { - await this.uploadDav(davPath, imagePath) + await this.upload(uploadPath, imagePath) } newLine = '![' + imageAlt + '](' + uploadedPath + '/' + imageName + ')' @@ -320,7 +317,7 @@ class Seance { }) if (!options.noUpload) { - this.uploadDav(davPath, imagePath) + this.upload(uploadPath, imagePath) } featuredImagePath = uploadedPath + '/' + imageName @@ -567,8 +564,7 @@ class Seance { loglevel: 'info' }) - await this.uploadDav(path.join(config.webdav.path_prefix,'avatars'), - filePath) + await this.upload('avatars', filePath) // Generate Ghost JSON @@ -581,7 +577,7 @@ class Seance { bio: json.payload.user.bio, email: email, name: json.payload.user.name, - profile_image: config.webdav.uploaded_path_prefix + '/avatars/' + fileName + profile_image: config.uploaded_path_prefix + '/avatars/' + fileName } ] }, @@ -680,6 +676,9 @@ class Seance { * @returns [string] status */ async uploadDav (dirPath, filePath) { + // Set uploadPath + // We'll do it directly since path.join mangles the protocol + let uploadPath = path.join(config.webdav.path_prefix, dirPath) // connect to webdav const client = createClient( @@ -692,7 +691,7 @@ class Seance { // create directory if not exists console.debug(`[dav-upload] Loading ${dirPath}`) - if (!await this.createDirIfNotExist(client, dirPath)) { + if (!await this.createDirIfNotExist(client, uploadPath)) { console.error(`[dav-upload] Could not upload ${path.basename(filePath)} :(`) return false } @@ -700,7 +699,7 @@ class Seance { // upload a file console.debug('Uploading file') const outStream = client.createWriteStream( - path.join(dirPath, path.basename(filePath)) + path.join(uploadPath, path.basename(filePath)) ) outStream.on('finish', () => console.debug('Uploaded successfully.')) @@ -710,6 +709,90 @@ class Seance { return true } + + /** + * function [exists] + * @returns [boolean] + * + * check if the given file exists or not + */ + async fsExists (path) { + try { + await fs.promises.access(path) + return true + } catch { + return false + } + } + + /** + * function [uploadLocal] + * @returns [string] status + * + * upload to a local file path. This should technically be + * called "copy" and not "upload", but the equivalent + * WebDAV one is actually an upload so ¯\_(ツ)_/¯ + */ + async uploadLocal (dirPath, filePath) { + // Set uploadPath + // We'll do it directly since path.join mangles the protocol + let uploadPath = path.join(config.local_upload.path_prefix, dirPath) + + // safety: don't touch directories outside LOCAL_UPLOAD_PATH_PREFIX + if (!uploadPath.startsWith(config.local_upload.path_prefix)) { + console.error(`[local-upload] Cannot create directories outside ${config.local_upload.path_prefix}`) + return false + } + + // create directory if not exists + console.debug(`[local-upload] Loading ${uploadPath}`) + if ( + !(await this.fsExists(uploadPath)) || + !(await fs.promises.lstat(uploadPath)).isDirectory() + ) { + if (!(await fs.promises.mkdir(uploadPath, + { recursive: true }))) { + console.error(`[local-upload] Could not upload ${path.basename(filePath)} :(`) + return false + } + } + + // actually do the copying + console.debug('Uploading file') + try { + await fs.promises.copyFile(filePath, + path.join(uploadPath, path.basename(filePath))) + } catch (err) { + console.error(`Upload error: ${err}`) + } + + return true + } + + + /** + * function [upload] + * @returns [boolean] status + * + * upload to WebDAV or a local folder, whichever is configured. + * If both are considered, WebDAV will be preferred. + */ + async upload (dirPath, filePath) { + if ( + !!config.webdav && + !!config.webdav.server_url && + !!config.webdav.path_prefix + ) { + return await this.uploadDav(dirPath, filePath) + } else if ( + !!config.local_upload && + !!config.local_upload.path_prefix + ) { + return await this.uploadLocal(dirPath, filePath) + } else { + throw { error: 'Either webdav or local_upload settings must be configured!' } + } + } } // Make Seance an EventEmitter