From 1db51d5776f6c9dce19968aa20f0903359fea8dc Mon Sep 17 00:00:00 2001 From: Sami Salkosuo Date: Mon, 20 Apr 2020 09:25:22 +0300 Subject: [PATCH] major refactoring --- src/app.js | 132 ++++++++++---------------------------- src/endpoints.js | 37 ----------- src/index.md | 18 +----- src/package.json | 1 + src/routes/convert.js | 105 ++++++++++++++++++++++++++++++ src/routes/test.js | 22 +++++++ src/routes/uploadfile.js | 88 +++++++++++++++++++++++++ src/{ => utils}/logger.js | 0 8 files changed, 251 insertions(+), 152 deletions(-) delete mode 100644 src/endpoints.js create mode 100644 src/routes/convert.js create mode 100644 src/routes/test.js create mode 100644 src/routes/uploadfile.js rename src/{ => utils}/logger.js (100%) diff --git a/src/app.js b/src/app.js index 19a24ab..feaf0e2 100644 --- a/src/app.js +++ b/src/app.js @@ -1,13 +1,9 @@ -const fs = require('fs'); const express = require('express'); const app = express(); -const Busboy = require('busboy'); const compression = require('compression'); -const ffmpeg = require('fluent-ffmpeg'); -const uniqueFilename = require('unique-filename'); -const endpoints = require('./endpoints.js'); +const all_routes = require('express-list-endpoints'); -const logger = require('./logger.js'); +const logger = require('./utils/logger.js'); const constants = require('./constants.js'); fileSizeLimit = constants.fileSizeLimit; @@ -27,103 +23,22 @@ process.on('SIGTERM', handle); app.use(compression()); -for (let prop in endpoints.types) { - if (endpoints.types.hasOwnProperty(prop)) { - let ffmpegParams = endpoints.types[prop]; - let bytes = 0; - app.post('/' + prop, function(req, res) { - let hitLimit = false; - let fileName = ''; - let savedFile = uniqueFilename('/tmp/'); - let busboy = new Busboy({ - headers: req.headers, - limits: { - files: 1, - fileSize: fileSizeLimit, - }}); - busboy.on('filesLimit', function() { - logger.error(`upload file size limit hit. max file size ${fileSizeLimit} bytes.`) - }); +//routes to handle file upload for all POST methods +var upload = require('./routes/uploadfile.js'); +app.use(upload); - busboy.on('file', function( - fieldname, - file, - filename, - encoding, - mimetype - ) { - file.on('limit', function(file) { - hitLimit = true; - let msg = `${filename} exceeds max size limit. max file size ${fileSizeLimit} bytes.` - logger.error(msg); - res.writeHead(500, {'Connection': 'close'}); - res.end(JSON.stringify({error: msg})); - }); - let log = { - file: filename, - encoding: encoding, - mimetype: mimetype, - }; - logger.debug(`file:${log.file}, encoding: ${log.encoding}, mimetype: ${log.mimetype}`); - file.on('data', function(data) { - bytes += data.length; - }); - file.on('end', function(data) { - log.bytes = bytes; - logger.debug(`file: ${log.file}, encoding: ${log.encoding}, mimetype: ${log.mimetype}, bytes: ${log.bytes}`); - }); +//test route for development +var test = require('./routes/test.js') +app.use('/test', test) - fileName = filename; - logger.debug(`uploading ${fileName}`) - let written = file.pipe(fs.createWriteStream(savedFile)); - if (written) { - logger.debug(`${fileName} saved, path: ${savedFile}`) - } - }); - busboy.on('finish', function() { - if (hitLimit) { - fs.unlinkSync(savedFile); - return; - } - logger.debug(`upload complete. file: ${fileName}`) - let outputFile = savedFile + '.' + ffmpegParams.extension; - logger.debug(`begin conversion from ${savedFile} to ${outputFile}`) - - //ffmpeg processing... - let ffmpegConvertCommand = ffmpeg(savedFile); - ffmpegConvertCommand - .renice(15) - .outputOptions(ffmpegParams.outputOptions) - .on('error', function(err) { - logger.error(`${err}`); - fs.unlinkSync(savedFile); - res.writeHead(500, {'Connection': 'close'}); - res.end(JSON.stringify({error: `${err}`})); - }) - .on('end', function() { - fs.unlinkSync(savedFile); - logger.debug(`starting download to client ${savedFile}`); +//routes to convert audio/video/image files to mp3/mp4/jpg +var convert = require('./routes/convert.js') +app.use('/convert', convert) - res.download(outputFile, null, function(err) { - if (err) { - logger.error(`download ${err}`); - } - logger.debug(`deleting ${outputFile}`); - if (fs.unlinkSync(outputFile)) { - logger.debug(`deleted ${outputFile}`); - } - }); - }) - .save(outputFile); - }); - return req.pipe(busboy); - }); - } -} require('express-readme')(app, { filename: 'index.md', - routes: ['/', '/readme'], + routes: ['/'], }); @@ -140,6 +55,25 @@ server.on('connection', function(socket) { server.keepAliveTimeout = timeout; }); -app.use(function(req, res, next) { - res.status(404).send(JSON.stringify({error: 'route not found'})+'\n'); +app.get('/endpoints', function(req, res) { + let code = 200; + res.writeHead(code, {'content-type' : 'text/plain'}); + res.end("Endpoints:\n\n"+JSON.stringify(all_routes(app),null,2)+'\n'); }); + +app.use(function(req, res, next) { + res.status(404).send({error: 'route not found'}); +}); + + +//custom error handler to return text/plain and message only +app.use(function(err, req, res, next){ + let code = err.statusCode || 500; + let message = err.message; + res.writeHead(code, {'content-type' : 'text/plain'}); + res.end(`${err.message}\n`); + +}); + + +logger.debug(JSON.stringify(all_routes(app))); diff --git a/src/endpoints.js b/src/endpoints.js deleted file mode 100644 index 039d4bb..0000000 --- a/src/endpoints.js +++ /dev/null @@ -1,37 +0,0 @@ -exports.types = { - jpg: { - extension: 'jpg', - outputOptions: [ - '-pix_fmt yuv422p', - ], - }, - m4a: { - extension: 'm4a', - outputOptions: [ - '-codec:a libfdk_aac', - ], - }, - mp3: { - extension: 'mp3', - outputOptions: [ - '-codec:a libmp3lame', - ], - }, - mp4: { - extension: 'mp4', - outputOptions: [ - '-codec:v libx264', - '-profile:v high', - '-r 15', - '-crf 23', - '-preset ultrafast', - '-b:v 500k', - '-maxrate 500k', - '-bufsize 1000k', - '-vf scale=-2:640', - '-threads 8', - '-codec:a libfdk_aac', - '-b:a 128k', - ], - }, -}; diff --git a/src/index.md b/src/index.md index 1f75bcb..ecae886 100644 --- a/src/index.md +++ b/src/index.md @@ -11,20 +11,6 @@ Based on: - https://github.com/fluent-ffmpeg/node-fluent-ffmpeg -### Endpoints - -- `POST /mp3` - Convert audio file in request body to mp3 -- `POST /mp4` - Convert video file in request body to mp4 -- `POST /jpg` - Convert image file to jpg -- `GET /` - Web Service Readme, this file - -### Examples - -- `curl -F "file=@input.wav" 127.0.0.1:3000/mp3 > output.mp3` -- `curl -F "file=@input.m4a" 127.0.0.1:3000/mp3 > output.mp3` -- `curl -F "file=@input.mov" 127.0.0.1:3000/mp4 > output.mp4` -- `curl -F "file=@input.mp4" 127.0.0.1:3000/mp4 > output.mp4` -- `curl -F "file=@input.tiff" 127.0.0.1:3000/jpg > output.jpg` -- `curl -F "file=@input.png" 127.0.0.1:3000/jpg > output.jpg` - +# Endpoints +[API endpoints](./endpoints) diff --git a/src/package.json b/src/package.json index a167565..5ad73f1 100644 --- a/src/package.json +++ b/src/package.json @@ -15,6 +15,7 @@ "express": "^4.17.1", "express-readme": "0.0.5", "fluent-ffmpeg": "^2.1.2", + "express-list-endpoints": "4.0.1", "fs": "0.0.1-security", "unique-filename": "^1.1.1", "winston": "^3.2.1" diff --git a/src/routes/convert.js b/src/routes/convert.js new file mode 100644 index 0000000..d133e39 --- /dev/null +++ b/src/routes/convert.js @@ -0,0 +1,105 @@ +var express = require('express') +const fs = require('fs'); +const ffmpeg = require('fluent-ffmpeg'); + +var router = express.Router() +const logger = require('../utils/logger.js') + + +//routes for /convert +//adds conversion type and format to res.locals. to be used in final post function +router.post('/audio/to/mp3', function (req, res,next) { + + res.locals.conversion="audio" + res.locals.format="mp3" + next() +}); + +router.post('/video/to/mp4', function (req, res,next) { + + res.locals.conversion="video" + res.locals.format="mp4" + next() +}); + +router.post('/image/to/jpg', function (req, res,next) { + + res.locals.conversion="image" + res.locals.format="jpg" + next() +}); + +// convert audio or video or image to mp3 or mp4 or jpg +router.post('*', function (req, res,next) { + let format = res.locals.format; + let conversion = res.locals.conversion; + logger.debug(`path: ${req.path}, conversion: ${conversion}, format: ${format}`); + if (conversion == undefined || format == undefined) + { + res.status(400).send("Invalid convert URL. Use one of: /convert/image/to/jpg, /convert/audio/to/mp3 or /convert/video/to/mp4.\n"); + return; + } + + let ffmpegParams ={ + extension: format + }; + if (conversion == "image") + { + ffmpegParams.outputOptions= ['-pix_fmt yuv422p'] + } + if (conversion == "audio") + { + ffmpegParams.outputOptions=['-codec:a libmp3lame' ] + } + if (conversion == "video") + { + ffmpegParams.outputOptions=[ + '-codec:v libx264', + '-profile:v high', + '-r 15', + '-crf 23', + '-preset ultrafast', + '-b:v 500k', + '-maxrate 500k', + '-bufsize 1000k', + '-vf scale=-2:640', + '-threads 8', + '-codec:a libfdk_aac', + '-b:a 128k', + ]; + } + + let savedFile = res.locals.savedFile; + let outputFile = savedFile + '-output.' + ffmpegParams.extension; + logger.debug(`begin conversion from ${savedFile} to ${outputFile}`) + + //ffmpeg processing... converting file... + let ffmpegConvertCommand = ffmpeg(savedFile); + ffmpegConvertCommand + .renice(15) + .outputOptions(ffmpegParams.outputOptions) + .on('error', function(err) { + logger.error(`${err}`); + fs.unlinkSync(savedFile); + res.writeHead(500, {'Connection': 'close'}); + res.end(JSON.stringify({error: `${err}`})); + }) + .on('end', function() { + fs.unlinkSync(savedFile); + logger.debug(`starting download to client ${savedFile}`); + + res.download(outputFile, null, function(err) { + if (err) { + logger.error(`download ${err}`); + } + logger.debug(`deleting ${outputFile}`); + if (fs.unlinkSync(outputFile)) { + logger.debug(`deleted ${outputFile}`); + } + }); + }) + .save(outputFile); + +}); + +module.exports = router \ No newline at end of file diff --git a/src/routes/test.js b/src/routes/test.js new file mode 100644 index 0000000..a82c8f3 --- /dev/null +++ b/src/routes/test.js @@ -0,0 +1,22 @@ +var express = require('express') + + +var router = express.Router() +const logger = require('../utils/logger.js') + +//route to handle file upload in all POST requests + + +// convert audio or video or image to mp3 or mp4 or jpg +router.post("/",function (req, res,next) { + logger.debug("path: " + req.path); + logger.debug("req.params: "); + for (const key in req.params) { + logger.debug(`${key}: ${req.params[key]}`); + } + logger.debug("res.locals.savedFile: " + res.locals.savedFile); + res.status(200).send("Test OK."); + +}); + +module.exports = router; \ No newline at end of file diff --git a/src/routes/uploadfile.js b/src/routes/uploadfile.js new file mode 100644 index 0000000..0672637 --- /dev/null +++ b/src/routes/uploadfile.js @@ -0,0 +1,88 @@ +var express = require('express') +const fs = require('fs'); +const Busboy = require('busboy'); +const uniqueFilename = require('unique-filename'); + +var router = express.Router() +const logger = require('../utils/logger.js') + +//route to handle file upload in all POST requests +router.use(function (req, res,next) { + + if(req.method == "POST") + { + logger.debug(`${__filename} path: ${req.path}`); + + let bytes = 0; + let hitLimit = false; + let fileName = ''; + var savedFile = uniqueFilename('/tmp/'); + let busboy = new Busboy({ + headers: req.headers, + limits: { + fields: 0, //no non-files allowed + files: 1, + fileSize: fileSizeLimit, + }}); + busboy.on('filesLimit', function() { + logger.error(`upload file size limit hit. max file size ${fileSizeLimit} bytes.`) + }); + busboy.on('fieldsLimit', function() { + let msg="Non-file field detected. Only files can be POSTed."; + logger.error(msg); + let err = new Error(msg); + err.statusCode = 400; + next(err); + }); + + busboy.on('file', function( + fieldname, + file, + filename, + encoding, + mimetype + ) { + file.on('limit', function(file) { + hitLimit = true; + let msg = `${filename} exceeds max size limit. max file size ${fileSizeLimit} bytes.` + logger.error(msg); + res.writeHead(500, {'Connection': 'close'}); + res.end(JSON.stringify({error: msg})); + }); + let log = { + file: filename, + encoding: encoding, + mimetype: mimetype, + }; + logger.debug(`file:${log.file}, encoding: ${log.encoding}, mimetype: ${log.mimetype}`); + file.on('data', function(data) { + bytes += data.length; + }); + file.on('end', function(data) { + log.bytes = bytes; + logger.debug(`file: ${log.file}, encoding: ${log.encoding}, mimetype: ${log.mimetype}, bytes: ${log.bytes}`); + }); + + fileName = filename; + savedFile = savedFile + "-" + fileName; + logger.debug(`uploading ${fileName}`) + let written = file.pipe(fs.createWriteStream(savedFile)); + if (written) { + logger.debug(`${fileName} saved, path: ${savedFile}`) + } + }); + busboy.on('finish', function() { + if (hitLimit) { + fs.unlinkSync(savedFile); + return; + } + logger.debug(`upload complete. file: ${fileName}`) + res.locals.savedFile = savedFile; + next(); + }); + return req.pipe(busboy); + } + next(); +}); + +module.exports = router; \ No newline at end of file diff --git a/src/logger.js b/src/utils/logger.js similarity index 100% rename from src/logger.js rename to src/utils/logger.js