From 7aa7e662b267e654db4f18fcbc0b9f30176f6cf1 Mon Sep 17 00:00:00 2001 From: advplyr Date: Wed, 6 Jul 2022 19:10:25 -0500 Subject: [PATCH] Remove dependency express-fileupload --- package-lock.json | 21 - package.json | 1 - server/Server.js | 2 +- server/libs/busboy/LICENSE | 19 + server/libs/busboy/index.js | 62 ++ server/libs/busboy/types/multipart.js | 657 ++++++++++++++++++ server/libs/busboy/types/urlencoded.js | 350 ++++++++++ server/libs/busboy/utils.js | 596 ++++++++++++++++ server/libs/expressFileupload/LICENSE | 21 + server/libs/expressFileupload/fileFactory.js | 65 ++ server/libs/expressFileupload/index.js | 39 ++ .../expressFileupload/isEligibleRequest.js | 34 + server/libs/expressFileupload/memHandler.js | 42 ++ .../expressFileupload/processMultipart.js | 168 +++++ .../libs/expressFileupload/processNested.js | 35 + .../libs/expressFileupload/tempFileHandler.js | 64 ++ server/libs/expressFileupload/uploadtimer.js | 26 + server/libs/expressFileupload/utilities.js | 311 +++++++++ server/libs/streamsearch/LICENSE | 19 + server/libs/streamsearch/index.js | 273 ++++++++ 20 files changed, 2782 insertions(+), 23 deletions(-) create mode 100644 server/libs/busboy/LICENSE create mode 100644 server/libs/busboy/index.js create mode 100644 server/libs/busboy/types/multipart.js create mode 100644 server/libs/busboy/types/urlencoded.js create mode 100644 server/libs/busboy/utils.js create mode 100644 server/libs/expressFileupload/LICENSE create mode 100644 server/libs/expressFileupload/fileFactory.js create mode 100644 server/libs/expressFileupload/index.js create mode 100644 server/libs/expressFileupload/isEligibleRequest.js create mode 100644 server/libs/expressFileupload/memHandler.js create mode 100644 server/libs/expressFileupload/processMultipart.js create mode 100644 server/libs/expressFileupload/processNested.js create mode 100644 server/libs/expressFileupload/tempFileHandler.js create mode 100644 server/libs/expressFileupload/uploadtimer.js create mode 100644 server/libs/expressFileupload/utilities.js create mode 100644 server/libs/streamsearch/LICENSE create mode 100644 server/libs/streamsearch/index.js diff --git a/package-lock.json b/package-lock.json index 74055677..01052b96 100644 --- a/package-lock.json +++ b/package-lock.json @@ -178,14 +178,6 @@ "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==" }, - "busboy": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", - "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", - "requires": { - "streamsearch": "^1.1.0" - } - }, "bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -441,14 +433,6 @@ "vary": "~1.1.2" } }, - "express-fileupload": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/express-fileupload/-/express-fileupload-1.4.0.tgz", - "integrity": "sha512-RjzLCHxkv3umDeZKeFeMg8w7qe0V09w3B7oGZprr/oO2H/ISCgNzuqzn7gV3HRWb37GjRk429CCpSLS2KNTqMQ==", - "requires": { - "busboy": "^1.6.0" - } - }, "express-rate-limit": { "version": "5.5.1", "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-5.5.1.tgz", @@ -945,11 +929,6 @@ "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==" }, - "streamsearch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", - "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==" - }, "string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", diff --git a/package.json b/package.json index 2a0b8e77..61d31ef5 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,6 @@ "axios": "^0.26.1", "date-and-time": "^2.3.1", "express": "^4.17.1", - "express-fileupload": "^1.2.1", "express-rate-limit": "^5.3.0", "htmlparser2": "^8.0.1", "socket.io": "^4.4.1", diff --git a/server/Server.js b/server/Server.js index 9484a43d..f43d46c7 100644 --- a/server/Server.js +++ b/server/Server.js @@ -3,7 +3,7 @@ const express = require('express') const http = require('http') const SocketIO = require('socket.io') const fs = require('./libs/fsExtra') -const fileUpload = require('express-fileupload') +const fileUpload = require('./libs/expressFileupload') const rateLimit = require('express-rate-limit') const { version } = require('../package.json') diff --git a/server/libs/busboy/LICENSE b/server/libs/busboy/LICENSE new file mode 100644 index 00000000..290762e9 --- /dev/null +++ b/server/libs/busboy/LICENSE @@ -0,0 +1,19 @@ +Copyright Brian White. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. \ No newline at end of file diff --git a/server/libs/busboy/index.js b/server/libs/busboy/index.js new file mode 100644 index 00000000..40d37c44 --- /dev/null +++ b/server/libs/busboy/index.js @@ -0,0 +1,62 @@ +'use strict'; + +// +// used by expressFileUpload +// Source: https://github.com/mscdex/busboy +// + +const { parseContentType } = require('./utils.js'); + +function getInstance(cfg) { + const headers = cfg.headers; + const conType = parseContentType(headers['content-type']); + if (!conType) + throw new Error('Malformed content type'); + + for (const type of TYPES) { + const matched = type.detect(conType); + if (!matched) + continue; + + const instanceCfg = { + limits: cfg.limits, + headers, + conType, + highWaterMark: undefined, + fileHwm: undefined, + defCharset: undefined, + defParamCharset: undefined, + preservePath: false, + }; + if (cfg.highWaterMark) + instanceCfg.highWaterMark = cfg.highWaterMark; + if (cfg.fileHwm) + instanceCfg.fileHwm = cfg.fileHwm; + instanceCfg.defCharset = cfg.defCharset; + instanceCfg.defParamCharset = cfg.defParamCharset; + instanceCfg.preservePath = cfg.preservePath; + return new type(instanceCfg); + } + + throw new Error(`Unsupported content type: ${headers['content-type']}`); +} + +// Note: types are explicitly listed here for easier bundling +// See: https://github.com/mscdex/busboy/issues/121 +const TYPES = [ + require('./types/multipart'), + require('./types/urlencoded'), +].filter(function (typemod) { return typeof typemod.detect === 'function'; }); + +module.exports = (cfg) => { + if (typeof cfg !== 'object' || cfg === null) + cfg = {}; + + if (typeof cfg.headers !== 'object' + || cfg.headers === null + || typeof cfg.headers['content-type'] !== 'string') { + throw new Error('Missing Content-Type'); + } + + return getInstance(cfg); +}; diff --git a/server/libs/busboy/types/multipart.js b/server/libs/busboy/types/multipart.js new file mode 100644 index 00000000..b8c04b13 --- /dev/null +++ b/server/libs/busboy/types/multipart.js @@ -0,0 +1,657 @@ +'use strict'; + +const { Readable, Writable } = require('stream'); + +const StreamSearch = require('../../streamsearch'); + +const { + basename, + convertToUTF8, + getDecoder, + parseContentType, + parseDisposition, +} = require('../utils.js'); + +const BUF_CRLF = Buffer.from('\r\n'); +const BUF_CR = Buffer.from('\r'); +const BUF_DASH = Buffer.from('-'); + +function noop() { } + +const MAX_HEADER_PAIRS = 2000; // From node +const MAX_HEADER_SIZE = 16 * 1024; // From node (its default value) + +const HPARSER_NAME = 0; +const HPARSER_PRE_OWS = 1; +const HPARSER_VALUE = 2; +class HeaderParser { + constructor(cb) { + this.header = Object.create(null); + this.pairCount = 0; + this.byteCount = 0; + this.state = HPARSER_NAME; + this.name = ''; + this.value = ''; + this.crlf = 0; + this.cb = cb; + } + + reset() { + this.header = Object.create(null); + this.pairCount = 0; + this.byteCount = 0; + this.state = HPARSER_NAME; + this.name = ''; + this.value = ''; + this.crlf = 0; + } + + push(chunk, pos, end) { + let start = pos; + while (pos < end) { + switch (this.state) { + case HPARSER_NAME: { + let done = false; + for (; pos < end; ++pos) { + if (this.byteCount === MAX_HEADER_SIZE) + return -1; + ++this.byteCount; + const code = chunk[pos]; + if (TOKEN[code] !== 1) { + if (code !== 58/* ':' */) + return -1; + this.name += chunk.latin1Slice(start, pos); + if (this.name.length === 0) + return -1; + ++pos; + done = true; + this.state = HPARSER_PRE_OWS; + break; + } + } + if (!done) { + this.name += chunk.latin1Slice(start, pos); + break; + } + // FALLTHROUGH + } + case HPARSER_PRE_OWS: { + // Skip optional whitespace + let done = false; + for (; pos < end; ++pos) { + if (this.byteCount === MAX_HEADER_SIZE) + return -1; + ++this.byteCount; + const code = chunk[pos]; + if (code !== 32/* ' ' */ && code !== 9/* '\t' */) { + start = pos; + done = true; + this.state = HPARSER_VALUE; + break; + } + } + if (!done) + break; + // FALLTHROUGH + } + case HPARSER_VALUE: + switch (this.crlf) { + case 0: // Nothing yet + for (; pos < end; ++pos) { + if (this.byteCount === MAX_HEADER_SIZE) + return -1; + ++this.byteCount; + const code = chunk[pos]; + if (FIELD_VCHAR[code] !== 1) { + if (code !== 13/* '\r' */) + return -1; + ++this.crlf; + break; + } + } + this.value += chunk.latin1Slice(start, pos++); + break; + case 1: // Received CR + if (this.byteCount === MAX_HEADER_SIZE) + return -1; + ++this.byteCount; + if (chunk[pos++] !== 10/* '\n' */) + return -1; + ++this.crlf; + break; + case 2: { // Received CR LF + if (this.byteCount === MAX_HEADER_SIZE) + return -1; + ++this.byteCount; + const code = chunk[pos]; + if (code === 32/* ' ' */ || code === 9/* '\t' */) { + // Folded value + start = pos; + this.crlf = 0; + } else { + if (++this.pairCount < MAX_HEADER_PAIRS) { + this.name = this.name.toLowerCase(); + if (this.header[this.name] === undefined) + this.header[this.name] = [this.value]; + else + this.header[this.name].push(this.value); + } + if (code === 13/* '\r' */) { + ++this.crlf; + ++pos; + } else { + // Assume start of next header field name + start = pos; + this.crlf = 0; + this.state = HPARSER_NAME; + this.name = ''; + this.value = ''; + } + } + break; + } + case 3: { // Received CR LF CR + if (this.byteCount === MAX_HEADER_SIZE) + return -1; + ++this.byteCount; + if (chunk[pos++] !== 10/* '\n' */) + return -1; + // End of header + const header = this.header; + this.reset(); + this.cb(header); + return pos; + } + } + break; + } + } + + return pos; + } +} + +class FileStream extends Readable { + constructor(opts, owner) { + super(opts); + this.truncated = false; + this._readcb = null; + this.once('end', () => { + // We need to make sure that we call any outstanding _writecb() that is + // associated with this file so that processing of the rest of the form + // can continue. This may not happen if the file stream ends right after + // backpressure kicks in, so we force it here. + this._read(); + if (--owner._fileEndsLeft === 0 && owner._finalcb) { + const cb = owner._finalcb; + owner._finalcb = null; + // Make sure other 'end' event handlers get a chance to be executed + // before busboy's 'finish' event is emitted + process.nextTick(cb); + } + }); + } + _read(n) { + const cb = this._readcb; + if (cb) { + this._readcb = null; + cb(); + } + } +} + +const ignoreData = { + push: (chunk, pos) => { }, + destroy: () => { }, +}; + +function callAndUnsetCb(self, err) { + const cb = self._writecb; + self._writecb = null; + if (err) + self.destroy(err); + else if (cb) + cb(); +} + +function nullDecoder(val, hint) { + return val; +} + +class Multipart extends Writable { + constructor(cfg) { + const streamOpts = { + autoDestroy: true, + emitClose: true, + highWaterMark: (typeof cfg.highWaterMark === 'number' + ? cfg.highWaterMark + : undefined), + }; + super(streamOpts); + + if (!cfg.conType.params || typeof cfg.conType.params.boundary !== 'string') + throw new Error('Multipart: Boundary not found'); + + const boundary = cfg.conType.params.boundary; + const paramDecoder = (typeof cfg.defParamCharset === 'string' + && cfg.defParamCharset + ? getDecoder(cfg.defParamCharset) + : nullDecoder); + const defCharset = (cfg.defCharset || 'utf8'); + const preservePath = cfg.preservePath; + const fileOpts = { + autoDestroy: true, + emitClose: true, + highWaterMark: (typeof cfg.fileHwm === 'number' + ? cfg.fileHwm + : undefined), + }; + + const limits = cfg.limits; + const fieldSizeLimit = (limits && typeof limits.fieldSize === 'number' + ? limits.fieldSize + : 1 * 1024 * 1024); + const fileSizeLimit = (limits && typeof limits.fileSize === 'number' + ? limits.fileSize + : Infinity); + const filesLimit = (limits && typeof limits.files === 'number' + ? limits.files + : Infinity); + const fieldsLimit = (limits && typeof limits.fields === 'number' + ? limits.fields + : Infinity); + const partsLimit = (limits && typeof limits.parts === 'number' + ? limits.parts + : Infinity); + + let parts = -1; // Account for initial boundary + let fields = 0; + let files = 0; + let skipPart = false; + + this._fileEndsLeft = 0; + this._fileStream = undefined; + this._complete = false; + let fileSize = 0; + + let field; + let fieldSize = 0; + let partCharset; + let partEncoding; + let partType; + let partName; + let partTruncated = false; + + let hitFilesLimit = false; + let hitFieldsLimit = false; + + this._hparser = null; + const hparser = new HeaderParser((header) => { + this._hparser = null; + skipPart = false; + + partType = 'text/plain'; + partCharset = defCharset; + partEncoding = '7bit'; + partName = undefined; + partTruncated = false; + + let filename; + if (!header['content-disposition']) { + skipPart = true; + return; + } + + const disp = parseDisposition(header['content-disposition'][0], + paramDecoder); + if (!disp || disp.type !== 'form-data') { + skipPart = true; + return; + } + + if (disp.params) { + if (disp.params.name) + partName = disp.params.name; + + if (disp.params['filename*']) + filename = disp.params['filename*']; + else if (disp.params.filename) + filename = disp.params.filename; + + if (filename !== undefined && !preservePath) + filename = basename(filename); + } + + if (header['content-type']) { + const conType = parseContentType(header['content-type'][0]); + if (conType) { + partType = `${conType.type}/${conType.subtype}`; + if (conType.params && typeof conType.params.charset === 'string') + partCharset = conType.params.charset.toLowerCase(); + } + } + + if (header['content-transfer-encoding']) + partEncoding = header['content-transfer-encoding'][0].toLowerCase(); + + if (partType === 'application/octet-stream' || filename !== undefined) { + // File + + if (files === filesLimit) { + if (!hitFilesLimit) { + hitFilesLimit = true; + this.emit('filesLimit'); + } + skipPart = true; + return; + } + ++files; + + if (this.listenerCount('file') === 0) { + skipPart = true; + return; + } + + fileSize = 0; + this._fileStream = new FileStream(fileOpts, this); + ++this._fileEndsLeft; + this.emit( + 'file', + partName, + this._fileStream, + { + filename, + encoding: partEncoding, + mimeType: partType + } + ); + } else { + // Non-file + + if (fields === fieldsLimit) { + if (!hitFieldsLimit) { + hitFieldsLimit = true; + this.emit('fieldsLimit'); + } + skipPart = true; + return; + } + ++fields; + + if (this.listenerCount('field') === 0) { + skipPart = true; + return; + } + + field = []; + fieldSize = 0; + } + }); + + let matchPostBoundary = 0; + const ssCb = (isMatch, data, start, end, isDataSafe) => { + retrydata: + while (data) { + if (this._hparser !== null) { + const ret = this._hparser.push(data, start, end); + if (ret === -1) { + this._hparser = null; + hparser.reset(); + this.emit('error', new Error('Malformed part header')); + break; + } + start = ret; + } + + if (start === end) + break; + + if (matchPostBoundary !== 0) { + if (matchPostBoundary === 1) { + switch (data[start]) { + case 45: // '-' + // Try matching '--' after boundary + matchPostBoundary = 2; + ++start; + break; + case 13: // '\r' + // Try matching CR LF before header + matchPostBoundary = 3; + ++start; + break; + default: + matchPostBoundary = 0; + } + if (start === end) + return; + } + + if (matchPostBoundary === 2) { + matchPostBoundary = 0; + if (data[start] === 45/* '-' */) { + // End of multipart data + this._complete = true; + this._bparser = ignoreData; + return; + } + // We saw something other than '-', so put the dash we consumed + // "back" + const writecb = this._writecb; + this._writecb = noop; + ssCb(false, BUF_DASH, 0, 1, false); + this._writecb = writecb; + } else if (matchPostBoundary === 3) { + matchPostBoundary = 0; + if (data[start] === 10/* '\n' */) { + ++start; + if (parts >= partsLimit) + break; + // Prepare the header parser + this._hparser = hparser; + if (start === end) + break; + // Process the remaining data as a header + continue retrydata; + } else { + // We saw something other than LF, so put the CR we consumed + // "back" + const writecb = this._writecb; + this._writecb = noop; + ssCb(false, BUF_CR, 0, 1, false); + this._writecb = writecb; + } + } + } + + if (!skipPart) { + if (this._fileStream) { + let chunk; + const actualLen = Math.min(end - start, fileSizeLimit - fileSize); + if (!isDataSafe) { + chunk = Buffer.allocUnsafe(actualLen); + data.copy(chunk, 0, start, start + actualLen); + } else { + chunk = data.slice(start, start + actualLen); + } + + fileSize += chunk.length; + if (fileSize === fileSizeLimit) { + if (chunk.length > 0) + this._fileStream.push(chunk); + this._fileStream.emit('limit'); + this._fileStream.truncated = true; + skipPart = true; + } else if (!this._fileStream.push(chunk)) { + if (this._writecb) + this._fileStream._readcb = this._writecb; + this._writecb = null; + } + } else if (field !== undefined) { + let chunk; + const actualLen = Math.min( + end - start, + fieldSizeLimit - fieldSize + ); + if (!isDataSafe) { + chunk = Buffer.allocUnsafe(actualLen); + data.copy(chunk, 0, start, start + actualLen); + } else { + chunk = data.slice(start, start + actualLen); + } + + fieldSize += actualLen; + field.push(chunk); + if (fieldSize === fieldSizeLimit) { + skipPart = true; + partTruncated = true; + } + } + } + + break; + } + + if (isMatch) { + matchPostBoundary = 1; + + if (this._fileStream) { + // End the active file stream if the previous part was a file + this._fileStream.push(null); + this._fileStream = null; + } else if (field !== undefined) { + let data; + switch (field.length) { + case 0: + data = ''; + break; + case 1: + data = convertToUTF8(field[0], partCharset, 0); + break; + default: + data = convertToUTF8( + Buffer.concat(field, fieldSize), + partCharset, + 0 + ); + } + field = undefined; + fieldSize = 0; + this.emit( + 'field', + partName, + data, + { + nameTruncated: false, + valueTruncated: partTruncated, + encoding: partEncoding, + mimeType: partType + } + ); + } + + if (++parts === partsLimit) + this.emit('partsLimit'); + } + }; + this._bparser = new StreamSearch(`\r\n--${boundary}`, ssCb); + + this._writecb = null; + this._finalcb = null; + + // Just in case there is no preamble + this.write(BUF_CRLF); + } + + static detect(conType) { + return (conType.type === 'multipart' && conType.subtype === 'form-data'); + } + + _write(chunk, enc, cb) { + this._writecb = cb; + this._bparser.push(chunk, 0); + if (this._writecb) + callAndUnsetCb(this); + } + + _destroy(err, cb) { + this._hparser = null; + this._bparser = ignoreData; + if (!err) + err = checkEndState(this); + const fileStream = this._fileStream; + if (fileStream) { + this._fileStream = null; + fileStream.destroy(err); + } + cb(err); + } + + _final(cb) { + this._bparser.destroy(); + if (!this._complete) + return cb(new Error('Unexpected end of form')); + if (this._fileEndsLeft) + this._finalcb = finalcb.bind(null, this, cb); + else + finalcb(this, cb); + } +} + +function finalcb(self, cb, err) { + if (err) + return cb(err); + err = checkEndState(self); + cb(err); +} + +function checkEndState(self) { + if (self._hparser) + return new Error('Malformed part header'); + const fileStream = self._fileStream; + if (fileStream) { + self._fileStream = null; + fileStream.destroy(new Error('Unexpected end of file')); + } + if (!self._complete) + return new Error('Unexpected end of form'); +} + +const TOKEN = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +]; + +const FIELD_VCHAR = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +]; + +module.exports = Multipart; diff --git a/server/libs/busboy/types/urlencoded.js b/server/libs/busboy/types/urlencoded.js new file mode 100644 index 00000000..5c463a25 --- /dev/null +++ b/server/libs/busboy/types/urlencoded.js @@ -0,0 +1,350 @@ +'use strict'; + +const { Writable } = require('stream'); + +const { getDecoder } = require('../utils.js'); + +class URLEncoded extends Writable { + constructor(cfg) { + const streamOpts = { + autoDestroy: true, + emitClose: true, + highWaterMark: (typeof cfg.highWaterMark === 'number' + ? cfg.highWaterMark + : undefined), + }; + super(streamOpts); + + let charset = (cfg.defCharset || 'utf8'); + if (cfg.conType.params && typeof cfg.conType.params.charset === 'string') + charset = cfg.conType.params.charset; + + this.charset = charset; + + const limits = cfg.limits; + this.fieldSizeLimit = (limits && typeof limits.fieldSize === 'number' + ? limits.fieldSize + : 1 * 1024 * 1024); + this.fieldsLimit = (limits && typeof limits.fields === 'number' + ? limits.fields + : Infinity); + this.fieldNameSizeLimit = ( + limits && typeof limits.fieldNameSize === 'number' + ? limits.fieldNameSize + : 100 + ); + + this._inKey = true; + this._keyTrunc = false; + this._valTrunc = false; + this._bytesKey = 0; + this._bytesVal = 0; + this._fields = 0; + this._key = ''; + this._val = ''; + this._byte = -2; + this._lastPos = 0; + this._encode = 0; + this._decoder = getDecoder(charset); + } + + static detect(conType) { + return (conType.type === 'application' + && conType.subtype === 'x-www-form-urlencoded'); + } + + _write(chunk, enc, cb) { + if (this._fields >= this.fieldsLimit) + return cb(); + + let i = 0; + const len = chunk.length; + this._lastPos = 0; + + // Check if we last ended mid-percent-encoded byte + if (this._byte !== -2) { + i = readPctEnc(this, chunk, i, len); + if (i === -1) + return cb(new Error('Malformed urlencoded form')); + if (i >= len) + return cb(); + if (this._inKey) + ++this._bytesKey; + else + ++this._bytesVal; + } + +main: + while (i < len) { + if (this._inKey) { + // Parsing key + + i = skipKeyBytes(this, chunk, i, len); + + while (i < len) { + switch (chunk[i]) { + case 61: // '=' + if (this._lastPos < i) + this._key += chunk.latin1Slice(this._lastPos, i); + this._lastPos = ++i; + this._key = this._decoder(this._key, this._encode); + this._encode = 0; + this._inKey = false; + continue main; + case 38: // '&' + if (this._lastPos < i) + this._key += chunk.latin1Slice(this._lastPos, i); + this._lastPos = ++i; + this._key = this._decoder(this._key, this._encode); + this._encode = 0; + if (this._bytesKey > 0) { + this.emit( + 'field', + this._key, + '', + { nameTruncated: this._keyTrunc, + valueTruncated: false, + encoding: this.charset, + mimeType: 'text/plain' } + ); + } + this._key = ''; + this._val = ''; + this._keyTrunc = false; + this._valTrunc = false; + this._bytesKey = 0; + this._bytesVal = 0; + if (++this._fields >= this.fieldsLimit) { + this.emit('fieldsLimit'); + return cb(); + } + continue; + case 43: // '+' + if (this._lastPos < i) + this._key += chunk.latin1Slice(this._lastPos, i); + this._key += ' '; + this._lastPos = i + 1; + break; + case 37: // '%' + if (this._encode === 0) + this._encode = 1; + if (this._lastPos < i) + this._key += chunk.latin1Slice(this._lastPos, i); + this._lastPos = i + 1; + this._byte = -1; + i = readPctEnc(this, chunk, i + 1, len); + if (i === -1) + return cb(new Error('Malformed urlencoded form')); + if (i >= len) + return cb(); + ++this._bytesKey; + i = skipKeyBytes(this, chunk, i, len); + continue; + } + ++i; + ++this._bytesKey; + i = skipKeyBytes(this, chunk, i, len); + } + if (this._lastPos < i) + this._key += chunk.latin1Slice(this._lastPos, i); + } else { + // Parsing value + + i = skipValBytes(this, chunk, i, len); + + while (i < len) { + switch (chunk[i]) { + case 38: // '&' + if (this._lastPos < i) + this._val += chunk.latin1Slice(this._lastPos, i); + this._lastPos = ++i; + this._inKey = true; + this._val = this._decoder(this._val, this._encode); + this._encode = 0; + if (this._bytesKey > 0 || this._bytesVal > 0) { + this.emit( + 'field', + this._key, + this._val, + { nameTruncated: this._keyTrunc, + valueTruncated: this._valTrunc, + encoding: this.charset, + mimeType: 'text/plain' } + ); + } + this._key = ''; + this._val = ''; + this._keyTrunc = false; + this._valTrunc = false; + this._bytesKey = 0; + this._bytesVal = 0; + if (++this._fields >= this.fieldsLimit) { + this.emit('fieldsLimit'); + return cb(); + } + continue main; + case 43: // '+' + if (this._lastPos < i) + this._val += chunk.latin1Slice(this._lastPos, i); + this._val += ' '; + this._lastPos = i + 1; + break; + case 37: // '%' + if (this._encode === 0) + this._encode = 1; + if (this._lastPos < i) + this._val += chunk.latin1Slice(this._lastPos, i); + this._lastPos = i + 1; + this._byte = -1; + i = readPctEnc(this, chunk, i + 1, len); + if (i === -1) + return cb(new Error('Malformed urlencoded form')); + if (i >= len) + return cb(); + ++this._bytesVal; + i = skipValBytes(this, chunk, i, len); + continue; + } + ++i; + ++this._bytesVal; + i = skipValBytes(this, chunk, i, len); + } + if (this._lastPos < i) + this._val += chunk.latin1Slice(this._lastPos, i); + } + } + + cb(); + } + + _final(cb) { + if (this._byte !== -2) + return cb(new Error('Malformed urlencoded form')); + if (!this._inKey || this._bytesKey > 0 || this._bytesVal > 0) { + if (this._inKey) + this._key = this._decoder(this._key, this._encode); + else + this._val = this._decoder(this._val, this._encode); + this.emit( + 'field', + this._key, + this._val, + { nameTruncated: this._keyTrunc, + valueTruncated: this._valTrunc, + encoding: this.charset, + mimeType: 'text/plain' } + ); + } + cb(); + } +} + +function readPctEnc(self, chunk, pos, len) { + if (pos >= len) + return len; + + if (self._byte === -1) { + // We saw a '%' but no hex characters yet + const hexUpper = HEX_VALUES[chunk[pos++]]; + if (hexUpper === -1) + return -1; + + if (hexUpper >= 8) + self._encode = 2; // Indicate high bits detected + + if (pos < len) { + // Both hex characters are in this chunk + const hexLower = HEX_VALUES[chunk[pos++]]; + if (hexLower === -1) + return -1; + + if (self._inKey) + self._key += String.fromCharCode((hexUpper << 4) + hexLower); + else + self._val += String.fromCharCode((hexUpper << 4) + hexLower); + + self._byte = -2; + self._lastPos = pos; + } else { + // Only one hex character was available in this chunk + self._byte = hexUpper; + } + } else { + // We saw only one hex character so far + const hexLower = HEX_VALUES[chunk[pos++]]; + if (hexLower === -1) + return -1; + + if (self._inKey) + self._key += String.fromCharCode((self._byte << 4) + hexLower); + else + self._val += String.fromCharCode((self._byte << 4) + hexLower); + + self._byte = -2; + self._lastPos = pos; + } + + return pos; +} + +function skipKeyBytes(self, chunk, pos, len) { + // Skip bytes if we've truncated + if (self._bytesKey > self.fieldNameSizeLimit) { + if (!self._keyTrunc) { + if (self._lastPos < pos) + self._key += chunk.latin1Slice(self._lastPos, pos - 1); + } + self._keyTrunc = true; + for (; pos < len; ++pos) { + const code = chunk[pos]; + if (code === 61/* '=' */ || code === 38/* '&' */) + break; + ++self._bytesKey; + } + self._lastPos = pos; + } + + return pos; +} + +function skipValBytes(self, chunk, pos, len) { + // Skip bytes if we've truncated + if (self._bytesVal > self.fieldSizeLimit) { + if (!self._valTrunc) { + if (self._lastPos < pos) + self._val += chunk.latin1Slice(self._lastPos, pos - 1); + } + self._valTrunc = true; + for (; pos < len; ++pos) { + if (chunk[pos] === 38/* '&' */) + break; + ++self._bytesVal; + } + self._lastPos = pos; + } + + return pos; +} + +/* eslint-disable no-multi-spaces */ +const HEX_VALUES = [ + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, +]; +/* eslint-enable no-multi-spaces */ + +module.exports = URLEncoded; diff --git a/server/libs/busboy/utils.js b/server/libs/busboy/utils.js new file mode 100644 index 00000000..8274f6c3 --- /dev/null +++ b/server/libs/busboy/utils.js @@ -0,0 +1,596 @@ +'use strict'; + +function parseContentType(str) { + if (str.length === 0) + return; + + const params = Object.create(null); + let i = 0; + + // Parse type + for (; i < str.length; ++i) { + const code = str.charCodeAt(i); + if (TOKEN[code] !== 1) { + if (code !== 47/* '/' */ || i === 0) + return; + break; + } + } + // Check for type without subtype + if (i === str.length) + return; + + const type = str.slice(0, i).toLowerCase(); + + // Parse subtype + const subtypeStart = ++i; + for (; i < str.length; ++i) { + const code = str.charCodeAt(i); + if (TOKEN[code] !== 1) { + // Make sure we have a subtype + if (i === subtypeStart) + return; + + if (parseContentTypeParams(str, i, params) === undefined) + return; + break; + } + } + // Make sure we have a subtype + if (i === subtypeStart) + return; + + const subtype = str.slice(subtypeStart, i).toLowerCase(); + + return { type, subtype, params }; +} + +function parseContentTypeParams(str, i, params) { + while (i < str.length) { + // Consume whitespace + for (; i < str.length; ++i) { + const code = str.charCodeAt(i); + if (code !== 32/* ' ' */ && code !== 9/* '\t' */) + break; + } + + // Ended on whitespace + if (i === str.length) + break; + + // Check for malformed parameter + if (str.charCodeAt(i++) !== 59/* ';' */) + return; + + // Consume whitespace + for (; i < str.length; ++i) { + const code = str.charCodeAt(i); + if (code !== 32/* ' ' */ && code !== 9/* '\t' */) + break; + } + + // Ended on whitespace (malformed) + if (i === str.length) + return; + + let name; + const nameStart = i; + // Parse parameter name + for (; i < str.length; ++i) { + const code = str.charCodeAt(i); + if (TOKEN[code] !== 1) { + if (code !== 61/* '=' */) + return; + break; + } + } + + // No value (malformed) + if (i === str.length) + return; + + name = str.slice(nameStart, i); + ++i; // Skip over '=' + + // No value (malformed) + if (i === str.length) + return; + + let value = ''; + let valueStart; + if (str.charCodeAt(i) === 34/* '"' */) { + valueStart = ++i; + let escaping = false; + // Parse quoted value + for (; i < str.length; ++i) { + const code = str.charCodeAt(i); + if (code === 92/* '\\' */) { + if (escaping) { + valueStart = i; + escaping = false; + } else { + value += str.slice(valueStart, i); + escaping = true; + } + continue; + } + if (code === 34/* '"' */) { + if (escaping) { + valueStart = i; + escaping = false; + continue; + } + value += str.slice(valueStart, i); + break; + } + if (escaping) { + valueStart = i - 1; + escaping = false; + } + // Invalid unescaped quoted character (malformed) + if (QDTEXT[code] !== 1) + return; + } + + // No end quote (malformed) + if (i === str.length) + return; + + ++i; // Skip over double quote + } else { + valueStart = i; + // Parse unquoted value + for (; i < str.length; ++i) { + const code = str.charCodeAt(i); + if (TOKEN[code] !== 1) { + // No value (malformed) + if (i === valueStart) + return; + break; + } + } + value = str.slice(valueStart, i); + } + + name = name.toLowerCase(); + if (params[name] === undefined) + params[name] = value; + } + + return params; +} + +function parseDisposition(str, defDecoder) { + if (str.length === 0) + return; + + const params = Object.create(null); + let i = 0; + + for (; i < str.length; ++i) { + const code = str.charCodeAt(i); + if (TOKEN[code] !== 1) { + if (parseDispositionParams(str, i, params, defDecoder) === undefined) + return; + break; + } + } + + const type = str.slice(0, i).toLowerCase(); + + return { type, params }; +} + +function parseDispositionParams(str, i, params, defDecoder) { + while (i < str.length) { + // Consume whitespace + for (; i < str.length; ++i) { + const code = str.charCodeAt(i); + if (code !== 32/* ' ' */ && code !== 9/* '\t' */) + break; + } + + // Ended on whitespace + if (i === str.length) + break; + + // Check for malformed parameter + if (str.charCodeAt(i++) !== 59/* ';' */) + return; + + // Consume whitespace + for (; i < str.length; ++i) { + const code = str.charCodeAt(i); + if (code !== 32/* ' ' */ && code !== 9/* '\t' */) + break; + } + + // Ended on whitespace (malformed) + if (i === str.length) + return; + + let name; + const nameStart = i; + // Parse parameter name + for (; i < str.length; ++i) { + const code = str.charCodeAt(i); + if (TOKEN[code] !== 1) { + if (code === 61/* '=' */) + break; + return; + } + } + + // No value (malformed) + if (i === str.length) + return; + + let value = ''; + let valueStart; + let charset; + //~ let lang; + name = str.slice(nameStart, i); + if (name.charCodeAt(name.length - 1) === 42/* '*' */) { + // Extended value + + const charsetStart = ++i; + // Parse charset name + for (; i < str.length; ++i) { + const code = str.charCodeAt(i); + if (CHARSET[code] !== 1) { + if (code !== 39/* '\'' */) + return; + break; + } + } + + // Incomplete charset (malformed) + if (i === str.length) + return; + + charset = str.slice(charsetStart, i); + ++i; // Skip over the '\'' + + //~ const langStart = ++i; + // Parse language name + for (; i < str.length; ++i) { + const code = str.charCodeAt(i); + if (code === 39/* '\'' */) + break; + } + + // Incomplete language (malformed) + if (i === str.length) + return; + + //~ lang = str.slice(langStart, i); + ++i; // Skip over the '\'' + + // No value (malformed) + if (i === str.length) + return; + + valueStart = i; + + let encode = 0; + // Parse value + for (; i < str.length; ++i) { + const code = str.charCodeAt(i); + if (EXTENDED_VALUE[code] !== 1) { + if (code === 37/* '%' */) { + let hexUpper; + let hexLower; + if (i + 2 < str.length + && (hexUpper = HEX_VALUES[str.charCodeAt(i + 1)]) !== -1 + && (hexLower = HEX_VALUES[str.charCodeAt(i + 2)]) !== -1) { + const byteVal = (hexUpper << 4) + hexLower; + value += str.slice(valueStart, i); + value += String.fromCharCode(byteVal); + i += 2; + valueStart = i + 1; + if (byteVal >= 128) + encode = 2; + else if (encode === 0) + encode = 1; + continue; + } + // '%' disallowed in non-percent encoded contexts (malformed) + return; + } + break; + } + } + + value += str.slice(valueStart, i); + value = convertToUTF8(value, charset, encode); + if (value === undefined) + return; + } else { + // Non-extended value + + ++i; // Skip over '=' + + // No value (malformed) + if (i === str.length) + return; + + if (str.charCodeAt(i) === 34/* '"' */) { + valueStart = ++i; + let escaping = false; + // Parse quoted value + for (; i < str.length; ++i) { + const code = str.charCodeAt(i); + if (code === 92/* '\\' */) { + if (escaping) { + valueStart = i; + escaping = false; + } else { + value += str.slice(valueStart, i); + escaping = true; + } + continue; + } + if (code === 34/* '"' */) { + if (escaping) { + valueStart = i; + escaping = false; + continue; + } + value += str.slice(valueStart, i); + break; + } + if (escaping) { + valueStart = i - 1; + escaping = false; + } + // Invalid unescaped quoted character (malformed) + if (QDTEXT[code] !== 1) + return; + } + + // No end quote (malformed) + if (i === str.length) + return; + + ++i; // Skip over double quote + } else { + valueStart = i; + // Parse unquoted value + for (; i < str.length; ++i) { + const code = str.charCodeAt(i); + if (TOKEN[code] !== 1) { + // No value (malformed) + if (i === valueStart) + return; + break; + } + } + value = str.slice(valueStart, i); + } + + value = defDecoder(value, 2); + if (value === undefined) + return; + } + + name = name.toLowerCase(); + if (params[name] === undefined) + params[name] = value; + } + + return params; +} + +function getDecoder(charset) { + let lc; + while (true) { + switch (charset) { + case 'utf-8': + case 'utf8': + return decoders.utf8; + case 'latin1': + case 'ascii': // TODO: Make these a separate, strict decoder? + case 'us-ascii': + case 'iso-8859-1': + case 'iso8859-1': + case 'iso88591': + case 'iso_8859-1': + case 'windows-1252': + case 'iso_8859-1:1987': + case 'cp1252': + case 'x-cp1252': + return decoders.latin1; + case 'utf16le': + case 'utf-16le': + case 'ucs2': + case 'ucs-2': + return decoders.utf16le; + case 'base64': + return decoders.base64; + default: + if (lc === undefined) { + lc = true; + charset = charset.toLowerCase(); + continue; + } + return decoders.other.bind(charset); + } + } +} + +const decoders = { + utf8: (data, hint) => { + if (data.length === 0) + return ''; + if (typeof data === 'string') { + // If `data` never had any percent-encoded bytes or never had any that + // were outside of the ASCII range, then we can safely just return the + // input since UTF-8 is ASCII compatible + if (hint < 2) + return data; + + data = Buffer.from(data, 'latin1'); + } + return data.utf8Slice(0, data.length); + }, + + latin1: (data, hint) => { + if (data.length === 0) + return ''; + if (typeof data === 'string') + return data; + return data.latin1Slice(0, data.length); + }, + + utf16le: (data, hint) => { + if (data.length === 0) + return ''; + if (typeof data === 'string') + data = Buffer.from(data, 'latin1'); + return data.ucs2Slice(0, data.length); + }, + + base64: (data, hint) => { + if (data.length === 0) + return ''; + if (typeof data === 'string') + data = Buffer.from(data, 'latin1'); + return data.base64Slice(0, data.length); + }, + + other: (data, hint) => { + if (data.length === 0) + return ''; + if (typeof data === 'string') + data = Buffer.from(data, 'latin1'); + try { + const decoder = new TextDecoder(this); + return decoder.decode(data); + } catch {} + }, +}; + +function convertToUTF8(data, charset, hint) { + const decode = getDecoder(charset); + if (decode) + return decode(data, hint); +} + +function basename(path) { + if (typeof path !== 'string') + return ''; + for (let i = path.length - 1; i >= 0; --i) { + switch (path.charCodeAt(i)) { + case 0x2F: // '/' + case 0x5C: // '\' + path = path.slice(i + 1); + return (path === '..' || path === '.' ? '' : path); + } + } + return (path === '..' || path === '.' ? '' : path); +} + +const TOKEN = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +]; + +const QDTEXT = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +]; + +const CHARSET = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +]; + +const EXTENDED_VALUE = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +]; + +/* eslint-disable no-multi-spaces */ +const HEX_VALUES = [ + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, +]; +/* eslint-enable no-multi-spaces */ + +module.exports = { + basename, + convertToUTF8, + getDecoder, + parseContentType, + parseDisposition, +}; diff --git a/server/libs/expressFileupload/LICENSE b/server/libs/expressFileupload/LICENSE new file mode 100644 index 00000000..7f6df6aa --- /dev/null +++ b/server/libs/expressFileupload/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015 Richard Girges + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/server/libs/expressFileupload/fileFactory.js b/server/libs/expressFileupload/fileFactory.js new file mode 100644 index 00000000..02b82df2 --- /dev/null +++ b/server/libs/expressFileupload/fileFactory.js @@ -0,0 +1,65 @@ +'use strict'; + +const { + isFunc, + debugLog, + moveFile, + promiseCallback, + checkAndMakeDir, + saveBufferToFile +} = require('./utilities'); + +/** + * Returns Local function that moves the file to a different location on the filesystem + * which takes two function arguments to make it compatible w/ Promise or Callback APIs + * @param {String} filePath - destination file path. + * @param {Object} options - file factory options. + * @param {Object} fileUploadOptions - middleware options. + * @returns {Function} + */ +const moveFromTemp = (filePath, options, fileUploadOptions) => (resolve, reject) => { + debugLog(fileUploadOptions, `Moving temporary file ${options.tempFilePath} to ${filePath}`); + moveFile(options.tempFilePath, filePath, promiseCallback(resolve, reject)); +}; + +/** + * Returns Local function that moves the file from buffer to a different location on the filesystem + * which takes two function arguments to make it compatible w/ Promise or Callback APIs + * @param {String} filePath - destination file path. + * @param {Object} options - file factory options. + * @param {Object} fileUploadOptions - middleware options. + * @returns {Function} + */ +const moveFromBuffer = (filePath, options, fileUploadOptions) => (resolve, reject) => { + debugLog(fileUploadOptions, `Moving uploaded buffer to ${filePath}`); + saveBufferToFile(options.buffer, filePath, promiseCallback(resolve, reject)); +}; + +module.exports = (options, fileUploadOptions = {}) => { + // see: https://github.com/richardgirges/express-fileupload/issues/14 + // firefox uploads empty file in case of cache miss when f5ing page. + // resulting in unexpected behavior. if there is no file data, the file is invalid. + // if (!fileUploadOptions.useTempFiles && !options.buffer.length) return; + + // Create and return file object. + return { + name: options.name, + data: options.buffer, + size: options.size, + encoding: options.encoding, + tempFilePath: options.tempFilePath, + truncated: options.truncated, + mimetype: options.mimetype, + md5: options.hash, + mv: (filePath, callback) => { + // Define a propper move function. + const moveFunc = fileUploadOptions.useTempFiles + ? moveFromTemp(filePath, options, fileUploadOptions) + : moveFromBuffer(filePath, options, fileUploadOptions); + // Create a folder for a file. + checkAndMakeDir(fileUploadOptions, filePath); + // If callback is passed in, use the callback API, otherwise return a promise. + return isFunc(callback) ? moveFunc(callback) : new Promise(moveFunc); + } + }; +}; diff --git a/server/libs/expressFileupload/index.js b/server/libs/expressFileupload/index.js new file mode 100644 index 00000000..95b39ba2 --- /dev/null +++ b/server/libs/expressFileupload/index.js @@ -0,0 +1,39 @@ +'use strict'; + +const path = require('path'); +const processMultipart = require('./processMultipart'); +const isEligibleRequest = require('./isEligibleRequest'); +const { buildOptions, debugLog } = require('./utilities'); +const busboy = require('../busboy'); // eslint-disable-line no-unused-vars + +const DEFAULT_OPTIONS = { + debug: false, + uploadTimeout: 60000, + fileHandler: false, + uriDecodeFileNames: false, + safeFileNames: false, + preserveExtension: false, + abortOnLimit: false, + responseOnLimit: 'File size limit has been reached', + limitHandler: false, + createParentPath: false, + parseNested: false, + useTempFiles: false, + tempFileDir: path.join(process.cwd(), 'tmp') +}; + +/** + * Expose the file upload middleware + * @param {DEFAULT_OPTIONS & busboy.BusboyConfig} options - Middleware options. + * @returns {Function} - express-fileupload middleware. + */ +module.exports = (options) => { + const uploadOptions = buildOptions(DEFAULT_OPTIONS, options); + return (req, res, next) => { + if (!isEligibleRequest(req)) { + debugLog(uploadOptions, 'Request is not eligible for file upload!'); + return next(); + } + processMultipart(uploadOptions, req, res, next); + }; +}; diff --git a/server/libs/expressFileupload/isEligibleRequest.js b/server/libs/expressFileupload/isEligibleRequest.js new file mode 100644 index 00000000..0c252833 --- /dev/null +++ b/server/libs/expressFileupload/isEligibleRequest.js @@ -0,0 +1,34 @@ +const ACCEPTABLE_CONTENT_TYPE = /^(multipart\/.+);(.*)$/i; +const UNACCEPTABLE_METHODS = ['GET', 'HEAD']; + +/** + * Ensures the request contains a content body + * @param {Object} req Express req object + * @returns {Boolean} + */ +const hasBody = (req) => { + return ('transfer-encoding' in req.headers) || + ('content-length' in req.headers && req.headers['content-length'] !== '0'); +}; + +/** + * Ensures the request is not using a non-compliant multipart method + * such as GET or HEAD + * @param {Object} req Express req object + * @returns {Boolean} + */ +const hasAcceptableMethod = req => !UNACCEPTABLE_METHODS.includes(req.method); + +/** + * Ensures that only multipart requests are processed by express-fileupload + * @param {Object} req Express req object + * @returns {Boolean} + */ +const hasAcceptableContentType = req => ACCEPTABLE_CONTENT_TYPE.test(req.headers['content-type']); + +/** + * Ensures that the request in question is eligible for file uploads + * @param {Object} req Express req object + * @returns {Boolean} + */ +module.exports = req => hasBody(req) && hasAcceptableMethod(req) && hasAcceptableContentType(req); diff --git a/server/libs/expressFileupload/memHandler.js b/server/libs/expressFileupload/memHandler.js new file mode 100644 index 00000000..09accfe0 --- /dev/null +++ b/server/libs/expressFileupload/memHandler.js @@ -0,0 +1,42 @@ +const crypto = require('crypto'); +const { debugLog } = require('./utilities'); + +/** + * memHandler - In memory upload handler + * @param {Object} options + * @param {String} fieldname + * @param {String} filename + * @returns {Object} + */ +module.exports = (options, fieldname, filename) => { + const buffers = []; + const hash = crypto.createHash('md5'); + let fileSize = 0; + let completed = false; + + const getBuffer = () => Buffer.concat(buffers, fileSize); + + return { + dataHandler: (data) => { + if (completed === true) { + debugLog(options, `Error: got ${fieldname}->${filename} data chunk for completed upload!`); + return; + } + buffers.push(data); + hash.update(data); + fileSize += data.length; + debugLog(options, `Uploading ${fieldname}->${filename}, bytes:${fileSize}...`); + }, + getBuffer: getBuffer, + getFilePath: () => '', + getFileSize: () => fileSize, + getHash: () => hash.digest('hex'), + complete: () => { + debugLog(options, `Upload ${fieldname}->${filename} completed, bytes:${fileSize}.`); + completed = true; + return getBuffer(); + }, + cleanup: () => { completed = true; }, + getWritePromise: () => Promise.resolve() + }; +}; diff --git a/server/libs/expressFileupload/processMultipart.js b/server/libs/expressFileupload/processMultipart.js new file mode 100644 index 00000000..32fb690a --- /dev/null +++ b/server/libs/expressFileupload/processMultipart.js @@ -0,0 +1,168 @@ +const Busboy = require('../busboy'); +const UploadTimer = require('./uploadtimer'); +const fileFactory = require('./fileFactory'); +const memHandler = require('./memHandler'); +const tempFileHandler = require('./tempFileHandler'); +const processNested = require('./processNested'); +const { + isFunc, + debugLog, + buildFields, + buildOptions, + parseFileName +} = require('./utilities'); + +const waitFlushProperty = Symbol('wait flush property symbol'); + +/** + * Processes multipart request + * Builds a req.body object for fields + * Builds a req.files object for files + * @param {Object} options expressFileupload and Busboy options + * @param {Object} req Express request object + * @param {Object} res Express response object + * @param {Function} next Express next method + * @return {void} + */ +module.exports = (options, req, res, next) => { + req.files = null; + + // Build busboy options and init busboy instance. + const busboyOptions = buildOptions(options, { headers: req.headers }); + const busboy = Busboy(busboyOptions); + + // Close connection with specified reason and http code, default: 400 Bad Request. + const closeConnection = (code, reason) => { + req.unpipe(busboy); + res.writeHead(code || 400, { Connection: 'close' }); + res.end(reason || 'Bad Request'); + }; + + // Express proxies sometimes attach multipart data to a buffer + if (req.body instanceof Buffer) { + req.body = Object.create(null); + } + // Build multipart req.body fields + busboy.on('field', (field, val) => req.body = buildFields(req.body, field, val)); + + // Build req.files fields + busboy.on('file', (field, file, info) => { + // Parse file name(cutting huge names, decoding, etc..). + const { filename: name, encoding, mimeType: mime } = info; + const filename = parseFileName(options, name); + // Define methods and handlers for upload process. + const { + dataHandler, + getFilePath, + getFileSize, + getHash, + complete, + cleanup, + getWritePromise + } = options.useTempFiles + ? tempFileHandler(options, field, filename) // Upload into temporary file. + : memHandler(options, field, filename); // Upload into RAM. + + const writePromise = options.useTempFiles + ? getWritePromise().catch(err => { + req.unpipe(busboy); + req.resume(); + cleanup(); + next(err); + }) : getWritePromise(); + + // Define upload timer. + const uploadTimer = new UploadTimer(options.uploadTimeout, () => { + file.removeAllListeners('data'); + file.resume(); + // After destroy an error event will be emitted and file clean up will be done. + file.destroy(new Error(`Upload timeout ${field}->${filename}, bytes:${getFileSize()}`)); + }); + + file.on('limit', () => { + debugLog(options, `Size limit reached for ${field}->${filename}, bytes:${getFileSize()}`); + // Reset upload timer in case of file limit reached. + uploadTimer.clear(); + // Run a user defined limit handler if it has been set. + if (isFunc(options.limitHandler)) return options.limitHandler(req, res, next); + // Close connection with 413 code and do cleanup if abortOnLimit set(default: false). + if (options.abortOnLimit) { + debugLog(options, `Aborting upload because of size limit ${field}->${filename}.`); + !isFunc(options.limitHandler) ? closeConnection(413, options.responseOnLimit) : ''; + cleanup(); + } + }); + + file.on('data', (data) => { + uploadTimer.set(); // Refresh upload timer each time new data chunk came. + dataHandler(data); // Handle new piece of data. + }); + + file.on('end', () => { + const size = getFileSize(); + // Debug logging for file upload ending. + debugLog(options, `Upload finished ${field}->${filename}, bytes:${size}`); + // Reset upload timer in case of end event. + uploadTimer.clear(); + // See https://github.com/richardgirges/express-fileupload/issues/191 + // Do not add file instance to the req.files if original name and size are empty. + // Empty name and zero size indicates empty file field in the posted form. + if (!name && size === 0) { + if (options.useTempFiles) { + cleanup(); + debugLog(options, `Removing the empty file ${field}->${filename}`); + } + return debugLog(options, `Don't add file instance if original name and size are empty`); + } + req.files = buildFields(req.files, field, fileFactory({ + buffer: complete(), + name: filename, + tempFilePath: getFilePath(), + hash: getHash(), + size, + encoding, + truncated: file.truncated, + mimetype: mime + }, options)); + + if (!req[waitFlushProperty]) { + req[waitFlushProperty] = []; + } + req[waitFlushProperty].push(writePromise); + }); + + file.on('error', (err) => { + uploadTimer.clear(); // Reset upload timer in case of errors. + debugLog(options, err); + cleanup(); + next(); + }); + + // Debug logging for a new file upload. + debugLog(options, `New upload started ${field}->${filename}, bytes:${getFileSize()}`); + // Set new upload timeout for a new file. + uploadTimer.set(); + }); + + busboy.on('finish', () => { + debugLog(options, `Busboy finished parsing request.`); + if (options.parseNested) { + req.body = processNested(req.body); + req.files = processNested(req.files); + } + + if (!req[waitFlushProperty]) return next(); + Promise.all(req[waitFlushProperty]) + .then(() => { + delete req[waitFlushProperty]; + next(); + }); + }); + + busboy.on('error', (err) => { + debugLog(options, `Busboy error`); + next(err); + }); + + req.pipe(busboy); +}; diff --git a/server/libs/expressFileupload/processNested.js b/server/libs/expressFileupload/processNested.js new file mode 100644 index 00000000..9c8b648e --- /dev/null +++ b/server/libs/expressFileupload/processNested.js @@ -0,0 +1,35 @@ +const { isSafeFromPollution } = require("./utilities"); + +module.exports = function(data){ + if (!data || data.length < 1) return Object.create(null); + + let d = Object.create(null), + keys = Object.keys(data); + + for (let i = 0; i < keys.length; i++) { + let key = keys[i], + value = data[key], + current = d, + keyParts = key + .replace(new RegExp(/\[/g), '.') + .replace(new RegExp(/\]/g), '') + .split('.'); + + for (let index = 0; index < keyParts.length; index++){ + let k = keyParts[index]; + + // Ensure we don't allow prototype pollution + if (!isSafeFromPollution(current, k)) { + continue; + } + + if (index >= keyParts.length - 1){ + current[k] = value; + } else { + if (!current[k]) current[k] = !isNaN(keyParts[index + 1]) ? [] : Object.create(null); + current = current[k]; + } + } + } + return d; +}; diff --git a/server/libs/expressFileupload/tempFileHandler.js b/server/libs/expressFileupload/tempFileHandler.js new file mode 100644 index 00000000..bfaca8b4 --- /dev/null +++ b/server/libs/expressFileupload/tempFileHandler.js @@ -0,0 +1,64 @@ +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); +const { + debugLog, + checkAndMakeDir, + getTempFilename, + deleteFile +} = require('./utilities'); + +module.exports = (options, fieldname, filename) => { + const dir = path.normalize(options.tempFileDir); + const tempFilePath = path.join(dir, getTempFilename()); + checkAndMakeDir({ createParentPath: true }, tempFilePath); + + debugLog(options, `Temporary file path is ${tempFilePath}`); + + const hash = crypto.createHash('md5'); + let fileSize = 0; + let completed = false; + + debugLog(options, `Opening write stream for ${fieldname}->${filename}...`); + const writeStream = fs.createWriteStream(tempFilePath); + const writePromise = new Promise((resolve, reject) => { + writeStream.on('finish', () => resolve()); + writeStream.on('error', (err) => { + debugLog(options, `Error write temp file: ${err}`); + reject(err); + }); + }); + + return { + dataHandler: (data) => { + if (completed === true) { + debugLog(options, `Error: got ${fieldname}->${filename} data chunk for completed upload!`); + return; + } + writeStream.write(data); + hash.update(data); + fileSize += data.length; + debugLog(options, `Uploading ${fieldname}->${filename}, bytes:${fileSize}...`); + }, + getFilePath: () => tempFilePath, + getFileSize: () => fileSize, + getHash: () => hash.digest('hex'), + complete: () => { + completed = true; + debugLog(options, `Upload ${fieldname}->${filename} completed, bytes:${fileSize}.`); + if (writeStream !== false) writeStream.end(); + // Return empty buff since data was uploaded into a temp file. + return Buffer.concat([]); + }, + cleanup: () => { + completed = true; + debugLog(options, `Cleaning up temporary file ${tempFilePath}...`); + writeStream.end(); + deleteFile(tempFilePath, err => (err + ? debugLog(options, `Cleaning up temporary file ${tempFilePath} failed: ${err}`) + : debugLog(options, `Cleaning up temporary file ${tempFilePath} done.`) + )); + }, + getWritePromise: () => writePromise + }; +}; diff --git a/server/libs/expressFileupload/uploadtimer.js b/server/libs/expressFileupload/uploadtimer.js new file mode 100644 index 00000000..d29ab46a --- /dev/null +++ b/server/libs/expressFileupload/uploadtimer.js @@ -0,0 +1,26 @@ +class UploadTimer { + /** + * @constructor + * @param {number} timeout - timer timeout in msecs. + * @param {Function} callback - callback to run when timeout reached. + */ + constructor(timeout = 0, callback = () => {}) { + this.timeout = timeout; + this.callback = callback; + this.timer = null; + } + + clear() { + clearTimeout(this.timer); + } + + set() { + // Do not start a timer if zero timeout or it hasn't been set. + if (!this.timeout) return false; + this.clear(); + this.timer = setTimeout(this.callback, this.timeout); + return true; + } +} + +module.exports = UploadTimer; diff --git a/server/libs/expressFileupload/utilities.js b/server/libs/expressFileupload/utilities.js new file mode 100644 index 00000000..b6bad01b --- /dev/null +++ b/server/libs/expressFileupload/utilities.js @@ -0,0 +1,311 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const { Readable } = require('stream'); + +// Parameters for safe file name parsing. +const SAFE_FILE_NAME_REGEX = /[^\w-]/g; +const MAX_EXTENSION_LENGTH = 3; + +// Parameters to generate unique temporary file names: +const TEMP_COUNTER_MAX = 65536; +const TEMP_PREFIX = 'tmp'; +let tempCounter = 0; + +/** + * Logs message to console if debug option set to true. + * @param {Object} options - options object. + * @param {string} msg - message to log. + * @returns {boolean} - false if debug is off. + */ +const debugLog = (options, msg) => { + const opts = options || {}; + if (!opts.debug) return false; + console.log(`Express-file-upload: ${msg}`); // eslint-disable-line + return true; +}; + +/** + * Generates unique temporary file name. e.g. tmp-5000-156788789789. + * @param {string} prefix - a prefix for generated unique file name. + * @returns {string} + */ +const getTempFilename = (prefix = TEMP_PREFIX) => { + tempCounter = tempCounter >= TEMP_COUNTER_MAX ? 1 : tempCounter + 1; + return `${prefix}-${tempCounter}-${Date.now()}`; +}; + +/** + * isFunc: Checks if argument is a function. + * @returns {boolean} - Returns true if argument is a function. + */ +const isFunc = func => func && func.constructor && func.call && func.apply ? true: false; + +/** + * Set errorFunc to the same value as successFunc for callback mode. + * @returns {Function} + */ +const errorFunc = (resolve, reject) => isFunc(reject) ? reject : resolve; + +/** + * Return a callback function for promise resole/reject args. + * Ensures that callback is called only once. + * @returns {Function} + */ +const promiseCallback = (resolve, reject) => { + let hasFired = false; + return (err) => { + if (hasFired) { + return; + } + + hasFired = true; + return err ? errorFunc(resolve, reject)(err) : resolve(); + }; +}; + +/** + * Builds instance options from arguments objects(can't be arrow function). + * @returns {Object} - result options. + */ +const buildOptions = function() { + const result = {}; + [...arguments].forEach(options => { + if (!options || typeof options !== 'object') return; + Object.keys(options).forEach(i => result[i] = options[i]); + }); + return result; +}; + +// The default prototypes for both objects and arrays. +// Used by isSafeFromPollution +const OBJECT_PROTOTYPE_KEYS = Object.getOwnPropertyNames(Object.prototype); +const ARRAY_PROTOTYPE_KEYS = Object.getOwnPropertyNames(Array.prototype); + +/** + * Determines whether a key insertion into an object could result in a prototype pollution + * @param {Object} base - The object whose insertion we are checking + * @param {string} key - The key that will be inserted + */ +const isSafeFromPollution = (base, key) => { + // We perform an instanceof check instead of Array.isArray as the former is more + // permissive for cases in which the object as an Array prototype but was not constructed + // via an Array constructor or literal. + const TOUCHES_ARRAY_PROTOTYPE = (base instanceof Array) && ARRAY_PROTOTYPE_KEYS.includes(key); + const TOUCHES_OBJECT_PROTOTYPE = OBJECT_PROTOTYPE_KEYS.includes(key); + + return !TOUCHES_ARRAY_PROTOTYPE && !TOUCHES_OBJECT_PROTOTYPE; +}; + +/** + * Builds request fields (using to build req.body and req.files) + * @param {Object} instance - request object. + * @param {string} field - field name. + * @param {any} value - field value. + * @returns {Object} + */ +const buildFields = (instance, field, value) => { + // Do nothing if value is not set. + if (value === null || value === undefined) return instance; + instance = instance || Object.create(null); + + if (!isSafeFromPollution(instance, field)) { + return instance; + } + // Non-array fields + if (!instance[field]) { + instance[field] = value; + return instance; + } + // Array fields + if (instance[field] instanceof Array) { + instance[field].push(value); + } else { + instance[field] = [instance[field], value]; + } + return instance; +}; + +/** + * Creates a folder for file specified in the path variable + * @param {Object} fileUploadOptions + * @param {string} filePath + * @returns {boolean} + */ +const checkAndMakeDir = (fileUploadOptions, filePath) => { + // Check upload options were set. + if (!fileUploadOptions) return false; + if (!fileUploadOptions.createParentPath) return false; + // Check whether folder for the file exists. + if (!filePath) return false; + const parentPath = path.dirname(filePath); + // Create folder if it doesn't exist. + if (!fs.existsSync(parentPath)) fs.mkdirSync(parentPath, { recursive: true }); + // Checks folder again and return a results. + return fs.existsSync(parentPath); +}; + +/** + * Deletes a file. + * @param {string} file - Path to the file to delete. + * @param {Function} callback + */ +const deleteFile = (file, callback) => fs.unlink(file, callback); + +/** + * Copy file via streams + * @param {string} src - Path to the source file + * @param {string} dst - Path to the destination file. + */ +const copyFile = (src, dst, callback) => { + // cbCalled flag and runCb helps to run cb only once. + let cbCalled = false; + let runCb = (err) => { + if (cbCalled) return; + cbCalled = true; + callback(err); + }; + // Create read stream + let readable = fs.createReadStream(src); + readable.on('error', runCb); + // Create write stream + let writable = fs.createWriteStream(dst); + writable.on('error', (err)=>{ + readable.destroy(); + runCb(err); + }); + writable.on('close', () => runCb()); + // Copy file via piping streams. + readable.pipe(writable); +}; + +/** + * moveFile: moves the file from src to dst. + * Firstly trying to rename the file if no luck copying it to dst and then deleteing src. + * @param {string} src - Path to the source file + * @param {string} dst - Path to the destination file. + * @param {Function} callback - A callback function. + */ +const moveFile = (src, dst, callback) => fs.rename(src, dst, err => (err + ? copyFile(src, dst, err => err ? callback(err) : deleteFile(src, callback)) + : callback() +)); + +/** + * Save buffer data to a file. + * @param {Buffer} buffer - buffer to save to a file. + * @param {string} filePath - path to a file. + */ +const saveBufferToFile = (buffer, filePath, callback) => { + if (!Buffer.isBuffer(buffer)) { + return callback(new Error('buffer variable should be type of Buffer!')); + } + // Setup readable stream from buffer. + let streamData = buffer; + let readStream = Readable(); + readStream._read = () => { + readStream.push(streamData); + streamData = null; + }; + // Setup file system writable stream. + let fstream = fs.createWriteStream(filePath); + // console.log("Calling saveBuffer"); + fstream.on('error', err => { + // console.log("err cb") + callback(err); + }); + fstream.on('close', () => { + // console.log("close cb"); + callback(); + }); + // Copy file via piping streams. + readStream.pipe(fstream); +}; + +/** + * Decodes uriEncoded file names. + * @param fileName {String} - file name to decode. + * @returns {String} + */ +const uriDecodeFileName = (opts, fileName) => { + return opts.uriDecodeFileNames ? decodeURIComponent(fileName) : fileName; +}; + +/** + * Parses filename and extension and returns object {name, extension}. + * @param {boolean|integer} preserveExtension - true/false or number of characters for extension. + * @param {string} fileName - file name to parse. + * @returns {Object} - { name, extension }. + */ +const parseFileNameExtension = (preserveExtension, fileName) => { + const preserveExtensionLength = parseInt(preserveExtension); + const result = {name: fileName, extension: ''}; + if (!preserveExtension && preserveExtensionLength !== 0) return result; + // Define maximum extension length + const maxExtLength = isNaN(preserveExtensionLength) + ? MAX_EXTENSION_LENGTH + : Math.abs(preserveExtensionLength); + + const nameParts = fileName.split('.'); + if (nameParts.length < 2) return result; + + let extension = nameParts.pop(); + if ( + extension.length > maxExtLength && + maxExtLength > 0 + ) { + nameParts[nameParts.length - 1] += + '.' + + extension.substr(0, extension.length - maxExtLength); + extension = extension.substr(-maxExtLength); + } + + result.extension = maxExtLength ? extension : ''; + result.name = nameParts.join('.'); + return result; +}; + +/** + * Parse file name and extension. + * @param {Object} opts - middleware options. + * @param {string} fileName - Uploaded file name. + * @returns {string} + */ +const parseFileName = (opts, fileName) => { + // Check fileName argument + if (!fileName || typeof fileName !== 'string') return getTempFilename(); + // Cut off file name if it's lenght more then 255. + let parsedName = fileName.length <= 255 ? fileName : fileName.substr(0, 255); + // Decode file name if uriDecodeFileNames option set true. + parsedName = uriDecodeFileName(opts, parsedName); + // Stop parsing file name if safeFileNames options hasn't been set. + if (!opts.safeFileNames) return parsedName; + // Set regular expression for the file name. + const nameRegex = typeof opts.safeFileNames === 'object' && opts.safeFileNames instanceof RegExp + ? opts.safeFileNames + : SAFE_FILE_NAME_REGEX; + // Parse file name extension. + let {name, extension} = parseFileNameExtension(opts.preserveExtension, parsedName); + if (extension.length) extension = '.' + extension.replace(nameRegex, ''); + + return name.replace(nameRegex, '').concat(extension); +}; + +module.exports = { + isFunc, + debugLog, + copyFile, // For testing purpose. + moveFile, + errorFunc, + deleteFile, // For testing purpose. + buildFields, + buildOptions, + parseFileName, + getTempFilename, + promiseCallback, + checkAndMakeDir, + saveBufferToFile, + uriDecodeFileName, + isSafeFromPollution +}; diff --git a/server/libs/streamsearch/LICENSE b/server/libs/streamsearch/LICENSE new file mode 100644 index 00000000..290762e9 --- /dev/null +++ b/server/libs/streamsearch/LICENSE @@ -0,0 +1,19 @@ +Copyright Brian White. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. \ No newline at end of file diff --git a/server/libs/streamsearch/index.js b/server/libs/streamsearch/index.js new file mode 100644 index 00000000..8683b58e --- /dev/null +++ b/server/libs/streamsearch/index.js @@ -0,0 +1,273 @@ +'use strict'; + +// +// used by busboy +// Source: https://github.com/mscdex/streamsearch +// + +/* + Based heavily on the Streaming Boyer-Moore-Horspool C++ implementation + by Hongli Lai at: https://github.com/FooBarWidget/boyer-moore-horspool +*/ +function memcmp(buf1, pos1, buf2, pos2, num) { + for (let i = 0; i < num; ++i) { + if (buf1[pos1 + i] !== buf2[pos2 + i]) + return false; + } + return true; +} + +class SBMH { + constructor(needle, cb) { + if (typeof cb !== 'function') + throw new Error('Missing match callback'); + + if (typeof needle === 'string') + needle = Buffer.from(needle); + else if (!Buffer.isBuffer(needle)) + throw new Error(`Expected Buffer for needle, got ${typeof needle}`); + + const needleLen = needle.length; + + this.maxMatches = Infinity; + this.matches = 0; + + this._cb = cb; + this._lookbehindSize = 0; + this._needle = needle; + this._bufPos = 0; + + this._lookbehind = Buffer.allocUnsafe(needleLen); + + // Initialize occurrence table. + this._occ = [ + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen, needleLen, needleLen, + needleLen, needleLen, needleLen, needleLen + ]; + + // Populate occurrence table with analysis of the needle, ignoring the last + // letter. + if (needleLen > 1) { + for (let i = 0; i < needleLen - 1; ++i) + this._occ[needle[i]] = needleLen - 1 - i; + } + } + + reset() { + this.matches = 0; + this._lookbehindSize = 0; + this._bufPos = 0; + } + + push(chunk, pos) { + let result; + if (!Buffer.isBuffer(chunk)) + chunk = Buffer.from(chunk, 'latin1'); + const chunkLen = chunk.length; + this._bufPos = pos || 0; + while (result !== chunkLen && this.matches < this.maxMatches) + result = feed(this, chunk); + return result; + } + + destroy() { + const lbSize = this._lookbehindSize; + if (lbSize) + this._cb(false, this._lookbehind, 0, lbSize, false); + this.reset(); + } +} + +function feed(self, data) { + const len = data.length; + const needle = self._needle; + const needleLen = needle.length; + + // Positive: points to a position in `data` + // pos == 3 points to data[3] + // Negative: points to a position in the lookbehind buffer + // pos == -2 points to lookbehind[lookbehindSize - 2] + let pos = -self._lookbehindSize; + const lastNeedleCharPos = needleLen - 1; + const lastNeedleChar = needle[lastNeedleCharPos]; + const end = len - needleLen; + const occ = self._occ; + const lookbehind = self._lookbehind; + + if (pos < 0) { + // Lookbehind buffer is not empty. Perform Boyer-Moore-Horspool + // search with character lookup code that considers both the + // lookbehind buffer and the current round's haystack data. + // + // Loop until + // there is a match. + // or until + // we've moved past the position that requires the + // lookbehind buffer. In this case we switch to the + // optimized loop. + // or until + // the character to look at lies outside the haystack. + while (pos < 0 && pos <= end) { + const nextPos = pos + lastNeedleCharPos; + const ch = (nextPos < 0 + ? lookbehind[self._lookbehindSize + nextPos] + : data[nextPos]); + + if (ch === lastNeedleChar + && matchNeedle(self, data, pos, lastNeedleCharPos)) { + self._lookbehindSize = 0; + ++self.matches; + if (pos > -self._lookbehindSize) + self._cb(true, lookbehind, 0, self._lookbehindSize + pos, false); + else + self._cb(true, undefined, 0, 0, true); + + return (self._bufPos = pos + needleLen); + } + + pos += occ[ch]; + } + + // No match. + + // There's too few data for Boyer-Moore-Horspool to run, + // so let's use a different algorithm to skip as much as + // we can. + // Forward pos until + // the trailing part of lookbehind + data + // looks like the beginning of the needle + // or until + // pos == 0 + while (pos < 0 && !matchNeedle(self, data, pos, len - pos)) + ++pos; + + if (pos < 0) { + // Cut off part of the lookbehind buffer that has + // been processed and append the entire haystack + // into it. + const bytesToCutOff = self._lookbehindSize + pos; + + if (bytesToCutOff > 0) { + // The cut off data is guaranteed not to contain the needle. + self._cb(false, lookbehind, 0, bytesToCutOff, false); + } + + self._lookbehindSize -= bytesToCutOff; + lookbehind.copy(lookbehind, 0, bytesToCutOff, self._lookbehindSize); + lookbehind.set(data, self._lookbehindSize); + self._lookbehindSize += len; + + self._bufPos = len; + return len; + } + + // Discard lookbehind buffer. + self._cb(false, lookbehind, 0, self._lookbehindSize, false); + self._lookbehindSize = 0; + } + + pos += self._bufPos; + + const firstNeedleChar = needle[0]; + + // Lookbehind buffer is now empty. Perform Boyer-Moore-Horspool + // search with optimized character lookup code that only considers + // the current round's haystack data. + while (pos <= end) { + const ch = data[pos + lastNeedleCharPos]; + + if (ch === lastNeedleChar + && data[pos] === firstNeedleChar + && memcmp(needle, 0, data, pos, lastNeedleCharPos)) { + ++self.matches; + if (pos > 0) + self._cb(true, data, self._bufPos, pos, true); + else + self._cb(true, undefined, 0, 0, true); + + return (self._bufPos = pos + needleLen); + } + + pos += occ[ch]; + } + + // There was no match. If there's trailing haystack data that we cannot + // match yet using the Boyer-Moore-Horspool algorithm (because the trailing + // data is less than the needle size) then match using a modified + // algorithm that starts matching from the beginning instead of the end. + // Whatever trailing data is left after running this algorithm is added to + // the lookbehind buffer. + while (pos < len) { + if (data[pos] !== firstNeedleChar + || !memcmp(data, pos, needle, 0, len - pos)) { + ++pos; + continue; + } + data.copy(lookbehind, 0, pos, len); + self._lookbehindSize = len - pos; + break; + } + + // Everything until `pos` is guaranteed not to contain needle data. + if (pos > 0) + self._cb(false, data, self._bufPos, pos < len ? pos : len, true); + + self._bufPos = len; + return len; +} + +function matchNeedle(self, data, pos, len) { + const lb = self._lookbehind; + const lbSize = self._lookbehindSize; + const needle = self._needle; + + for (let i = 0; i < len; ++i, ++pos) { + const ch = (pos < 0 ? lb[lbSize + pos] : data[pos]); + if (ch !== needle[i]) + return false; + } + return true; +} + +module.exports = SBMH; \ No newline at end of file