diff --git a/package-lock.json b/package-lock.json index 8eea03ce..b5104019 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,7 +24,6 @@ "libgen": "^2.1.0", "node-ffprobe": "^3.0.0", "node-stream-zip": "^1.15.0", - "proper-lockfile": "^4.1.2", "recursive-readdir-async": "^1.1.8", "socket.io": "^4.4.1", "xml2js": "^0.4.23" @@ -1488,16 +1487,6 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, - "node_modules/proper-lockfile": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", - "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", - "dependencies": { - "graceful-fs": "^4.2.4", - "retry": "^0.12.0", - "signal-exit": "^3.0.2" - } - }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -1608,14 +1597,6 @@ "lowercase-keys": "^2.0.0" } }, - "node_modules/retry": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", - "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", - "engines": { - "node": ">= 4" - } - }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -1713,11 +1694,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/signal-exit": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" - }, "node_modules/socket.io": { "version": "4.5.1", "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.5.1.tgz", @@ -3084,16 +3060,6 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, - "proper-lockfile": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", - "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", - "requires": { - "graceful-fs": "^4.2.4", - "retry": "^0.12.0", - "signal-exit": "^3.0.2" - } - }, "proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -3177,11 +3143,6 @@ "lowercase-keys": "^2.0.0" } }, - "retry": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", - "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==" - }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -3255,11 +3216,6 @@ "object-inspect": "^1.9.0" } }, - "signal-exit": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" - }, "socket.io": { "version": "4.5.1", "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.5.1.tgz", diff --git a/package.json b/package.json index 2bd2201b..b19bcd3d 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,6 @@ "libgen": "^2.1.0", "node-ffprobe": "^3.0.0", "node-stream-zip": "^1.15.0", - "proper-lockfile": "^4.1.2", "recursive-readdir-async": "^1.1.8", "socket.io": "^4.4.1", "xml2js": "^0.4.23" diff --git a/server/Db.js b/server/Db.js index 37e54eb0..f9fac6d4 100644 --- a/server/Db.js +++ b/server/Db.js @@ -1,6 +1,5 @@ const Path = require('path') -const njodb = require('./njodb') -const jwt = require('jsonwebtoken') +const njodb = require('./libs/njodb') const Logger = require('./Logger') const { version } = require('../package.json') const LibraryItem = require('./objects/LibraryItem') diff --git a/server/njodb/index.js b/server/libs/njodb/index.js similarity index 100% rename from server/njodb/index.js rename to server/libs/njodb/index.js diff --git a/server/njodb/njodb.js b/server/libs/njodb/njodb.js similarity index 99% rename from server/njodb/njodb.js rename to server/libs/njodb/njodb.js index e8639aa0..f61d8b73 100644 --- a/server/njodb/njodb.js +++ b/server/libs/njodb/njodb.js @@ -27,7 +27,7 @@ const { checkSync, lock, lockSync -} = require("proper-lockfile"); +} = require("../properLockfile"); const { deleteFile, diff --git a/server/njodb/objects.js b/server/libs/njodb/objects.js similarity index 100% rename from server/njodb/objects.js rename to server/libs/njodb/objects.js diff --git a/server/njodb/utils.js b/server/libs/njodb/utils.js similarity index 100% rename from server/njodb/utils.js rename to server/libs/njodb/utils.js diff --git a/server/njodb/validators.js b/server/libs/njodb/validators.js similarity index 100% rename from server/njodb/validators.js rename to server/libs/njodb/validators.js diff --git a/server/libs/properLockfile/index.js b/server/libs/properLockfile/index.js new file mode 100644 index 00000000..68cbba18 --- /dev/null +++ b/server/libs/properLockfile/index.js @@ -0,0 +1,40 @@ +'use strict'; + +const lockfile = require('./lib/lockfile'); +const { toPromise, toSync, toSyncOptions } = require('./lib/adapter'); + +async function lock(file, options) { + const release = await toPromise(lockfile.lock)(file, options); + + return toPromise(release); +} + +function lockSync(file, options) { + const release = toSync(lockfile.lock)(file, toSyncOptions(options)); + + return toSync(release); +} + +function unlock(file, options) { + return toPromise(lockfile.unlock)(file, options); +} + +function unlockSync(file, options) { + return toSync(lockfile.unlock)(file, toSyncOptions(options)); +} + +function check(file, options) { + return toPromise(lockfile.check)(file, options); +} + +function checkSync(file, options) { + return toSync(lockfile.check)(file, toSyncOptions(options)); +} + +module.exports = lock; +module.exports.lock = lock; +module.exports.unlock = unlock; +module.exports.lockSync = lockSync; +module.exports.unlockSync = unlockSync; +module.exports.check = check; +module.exports.checkSync = checkSync; diff --git a/server/libs/properLockfile/lib/adapter.js b/server/libs/properLockfile/lib/adapter.js new file mode 100644 index 00000000..85789672 --- /dev/null +++ b/server/libs/properLockfile/lib/adapter.js @@ -0,0 +1,85 @@ +'use strict'; + +const fs = require('graceful-fs'); + +function createSyncFs(fs) { + const methods = ['mkdir', 'realpath', 'stat', 'rmdir', 'utimes']; + const newFs = { ...fs }; + + methods.forEach((method) => { + newFs[method] = (...args) => { + const callback = args.pop(); + let ret; + + try { + ret = fs[`${method}Sync`](...args); + } catch (err) { + return callback(err); + } + + callback(null, ret); + }; + }); + + return newFs; +} + +// ---------------------------------------------------------- + +function toPromise(method) { + return (...args) => new Promise((resolve, reject) => { + args.push((err, result) => { + if (err) { + reject(err); + } else { + resolve(result); + } + }); + + method(...args); + }); +} + +function toSync(method) { + return (...args) => { + let err; + let result; + + args.push((_err, _result) => { + err = _err; + result = _result; + }); + + method(...args); + + if (err) { + throw err; + } + + return result; + }; +} + +function toSyncOptions(options) { + // Shallow clone options because we are oging to mutate them + options = { ...options }; + + // Transform fs to use the sync methods instead + options.fs = createSyncFs(options.fs || fs); + + // Retries are not allowed because it requires the flow to be sync + if ( + (typeof options.retries === 'number' && options.retries > 0) || + (options.retries && typeof options.retries.retries === 'number' && options.retries.retries > 0) + ) { + throw Object.assign(new Error('Cannot use retries with the sync api'), { code: 'ESYNC' }); + } + + return options; +} + +module.exports = { + toPromise, + toSync, + toSyncOptions, +}; diff --git a/server/libs/properLockfile/lib/lockfile.js b/server/libs/properLockfile/lib/lockfile.js new file mode 100644 index 00000000..d981a00c --- /dev/null +++ b/server/libs/properLockfile/lib/lockfile.js @@ -0,0 +1,342 @@ +'use strict'; + +const path = require('path'); +const fs = require('graceful-fs'); +const retry = require('../../retry'); +const onExit = require('../../signalExit'); +const mtimePrecision = require('./mtime-precision'); + +const locks = {}; + +function getLockFile(file, options) { + return options.lockfilePath || `${file}.lock`; +} + +function resolveCanonicalPath(file, options, callback) { + if (!options.realpath) { + return callback(null, path.resolve(file)); + } + + // Use realpath to resolve symlinks + // It also resolves relative paths + options.fs.realpath(file, callback); +} + +function acquireLock(file, options, callback) { + const lockfilePath = getLockFile(file, options); + + // Use mkdir to create the lockfile (atomic operation) + options.fs.mkdir(lockfilePath, (err) => { + if (!err) { + // At this point, we acquired the lock! + // Probe the mtime precision + return mtimePrecision.probe(lockfilePath, options.fs, (err, mtime, mtimePrecision) => { + // If it failed, try to remove the lock.. + /* istanbul ignore if */ + if (err) { + options.fs.rmdir(lockfilePath, () => { }); + + return callback(err); + } + + callback(null, mtime, mtimePrecision); + }); + } + + // If error is not EEXIST then some other error occurred while locking + if (err.code !== 'EEXIST') { + return callback(err); + } + + // Otherwise, check if lock is stale by analyzing the file mtime + if (options.stale <= 0) { + return callback(Object.assign(new Error('Lock file is already being held'), { code: 'ELOCKED', file })); + } + + options.fs.stat(lockfilePath, (err, stat) => { + if (err) { + // Retry if the lockfile has been removed (meanwhile) + // Skip stale check to avoid recursiveness + if (err.code === 'ENOENT') { + return acquireLock(file, { ...options, stale: 0 }, callback); + } + + return callback(err); + } + + if (!isLockStale(stat, options)) { + return callback(Object.assign(new Error('Lock file is already being held'), { code: 'ELOCKED', file })); + } + + // If it's stale, remove it and try again! + // Skip stale check to avoid recursiveness + removeLock(file, options, (err) => { + if (err) { + return callback(err); + } + + acquireLock(file, { ...options, stale: 0 }, callback); + }); + }); + }); +} + +function isLockStale(stat, options) { + return stat.mtime.getTime() < Date.now() - options.stale; +} + +function removeLock(file, options, callback) { + // Remove lockfile, ignoring ENOENT errors + options.fs.rmdir(getLockFile(file, options), (err) => { + if (err && err.code !== 'ENOENT') { + return callback(err); + } + + callback(); + }); +} + +function updateLock(file, options) { + const lock = locks[file]; + + // Just for safety, should never happen + /* istanbul ignore if */ + if (lock.updateTimeout) { + return; + } + + lock.updateDelay = lock.updateDelay || options.update; + lock.updateTimeout = setTimeout(() => { + lock.updateTimeout = null; + + // Stat the file to check if mtime is still ours + // If it is, we can still recover from a system sleep or a busy event loop + options.fs.stat(lock.lockfilePath, (err, stat) => { + const isOverThreshold = lock.lastUpdate + options.stale < Date.now(); + + // If it failed to update the lockfile, keep trying unless + // the lockfile was deleted or we are over the threshold + if (err) { + if (err.code === 'ENOENT' || isOverThreshold) { + return setLockAsCompromised(file, lock, Object.assign(err, { code: 'ECOMPROMISED' })); + } + + lock.updateDelay = 1000; + + return updateLock(file, options); + } + + const isMtimeOurs = lock.mtime.getTime() === stat.mtime.getTime(); + + if (!isMtimeOurs) { + return setLockAsCompromised( + file, + lock, + Object.assign( + new Error('Unable to update lock within the stale threshold'), + { code: 'ECOMPROMISED' } + )); + } + + const mtime = mtimePrecision.getMtime(lock.mtimePrecision); + + options.fs.utimes(lock.lockfilePath, mtime, mtime, (err) => { + const isOverThreshold = lock.lastUpdate + options.stale < Date.now(); + + // Ignore if the lock was released + if (lock.released) { + return; + } + + // If it failed to update the lockfile, keep trying unless + // the lockfile was deleted or we are over the threshold + if (err) { + if (err.code === 'ENOENT' || isOverThreshold) { + return setLockAsCompromised(file, lock, Object.assign(err, { code: 'ECOMPROMISED' })); + } + + lock.updateDelay = 1000; + + return updateLock(file, options); + } + + // All ok, keep updating.. + lock.mtime = mtime; + lock.lastUpdate = Date.now(); + lock.updateDelay = null; + updateLock(file, options); + }); + }); + }, lock.updateDelay); + + // Unref the timer so that the nodejs process can exit freely + // This is safe because all acquired locks will be automatically released + // on process exit + + // We first check that `lock.updateTimeout.unref` exists because some users + // may be using this module outside of NodeJS (e.g., in an electron app), + // and in those cases `setTimeout` return an integer. + /* istanbul ignore else */ + if (lock.updateTimeout.unref) { + lock.updateTimeout.unref(); + } +} + +function setLockAsCompromised(file, lock, err) { + // Signal the lock has been released + lock.released = true; + + // Cancel lock mtime update + // Just for safety, at this point updateTimeout should be null + /* istanbul ignore if */ + if (lock.updateTimeout) { + clearTimeout(lock.updateTimeout); + } + + if (locks[file] === lock) { + delete locks[file]; + } + + lock.options.onCompromised(err); +} + +// ---------------------------------------------------------- + +function lock(file, options, callback) { + /* istanbul ignore next */ + options = { + stale: 10000, + update: null, + realpath: true, + retries: 0, + fs, + onCompromised: (err) => { throw err; }, + ...options, + }; + + options.retries = options.retries || 0; + options.retries = typeof options.retries === 'number' ? { retries: options.retries } : options.retries; + options.stale = Math.max(options.stale || 0, 2000); + options.update = options.update == null ? options.stale / 2 : options.update || 0; + options.update = Math.max(Math.min(options.update, options.stale / 2), 1000); + + // Resolve to a canonical file path + resolveCanonicalPath(file, options, (err, file) => { + if (err) { + return callback(err); + } + + // Attempt to acquire the lock + const operation = retry.operation(options.retries); + + operation.attempt(() => { + acquireLock(file, options, (err, mtime, mtimePrecision) => { + if (operation.retry(err)) { + return; + } + + if (err) { + return callback(operation.mainError()); + } + + // We now own the lock + const lock = locks[file] = { + lockfilePath: getLockFile(file, options), + mtime, + mtimePrecision, + options, + lastUpdate: Date.now(), + }; + + // We must keep the lock fresh to avoid staleness + updateLock(file, options); + + callback(null, (releasedCallback) => { + if (lock.released) { + return releasedCallback && + releasedCallback(Object.assign(new Error('Lock is already released'), { code: 'ERELEASED' })); + } + + // Not necessary to use realpath twice when unlocking + unlock(file, { ...options, realpath: false }, releasedCallback); + }); + }); + }); + }); +} + +function unlock(file, options, callback) { + options = { + fs, + realpath: true, + ...options, + }; + + // Resolve to a canonical file path + resolveCanonicalPath(file, options, (err, file) => { + if (err) { + return callback(err); + } + + // Skip if the lock is not acquired + const lock = locks[file]; + + if (!lock) { + return callback(Object.assign(new Error('Lock is not acquired/owned by you'), { code: 'ENOTACQUIRED' })); + } + + lock.updateTimeout && clearTimeout(lock.updateTimeout); // Cancel lock mtime update + lock.released = true; // Signal the lock has been released + delete locks[file]; // Delete from locks + + removeLock(file, options, callback); + }); +} + +function check(file, options, callback) { + options = { + stale: 10000, + realpath: true, + fs, + ...options, + }; + + options.stale = Math.max(options.stale || 0, 2000); + + // Resolve to a canonical file path + resolveCanonicalPath(file, options, (err, file) => { + if (err) { + return callback(err); + } + + // Check if lockfile exists + options.fs.stat(getLockFile(file, options), (err, stat) => { + if (err) { + // If does not exist, file is not locked. Otherwise, callback with error + return err.code === 'ENOENT' ? callback(null, false) : callback(err); + } + + // Otherwise, check if lock is stale by analyzing the file mtime + return callback(null, !isLockStale(stat, options)); + }); + }); +} + +function getLocks() { + return locks; +} + +// Remove acquired locks on exit +/* istanbul ignore next */ +onExit(() => { + for (const file in locks) { + const options = locks[file].options; + + try { options.fs.rmdirSync(getLockFile(file, options)); } catch (e) { /* Empty */ } + } +}); + +module.exports.lock = lock; +module.exports.unlock = unlock; +module.exports.check = check; +module.exports.getLocks = getLocks; diff --git a/server/libs/properLockfile/lib/mtime-precision.js b/server/libs/properLockfile/lib/mtime-precision.js new file mode 100644 index 00000000..b82a3ce0 --- /dev/null +++ b/server/libs/properLockfile/lib/mtime-precision.js @@ -0,0 +1,55 @@ +'use strict'; + +const cacheSymbol = Symbol(); + +function probe(file, fs, callback) { + const cachedPrecision = fs[cacheSymbol]; + + if (cachedPrecision) { + return fs.stat(file, (err, stat) => { + /* istanbul ignore if */ + if (err) { + return callback(err); + } + + callback(null, stat.mtime, cachedPrecision); + }); + } + + // Set mtime by ceiling Date.now() to seconds + 5ms so that it's "not on the second" + const mtime = new Date((Math.ceil(Date.now() / 1000) * 1000) + 5); + + fs.utimes(file, mtime, mtime, (err) => { + /* istanbul ignore if */ + if (err) { + return callback(err); + } + + fs.stat(file, (err, stat) => { + /* istanbul ignore if */ + if (err) { + return callback(err); + } + + const precision = stat.mtime.getTime() % 1000 === 0 ? 's' : 'ms'; + + // Cache the precision in a non-enumerable way + Object.defineProperty(fs, cacheSymbol, { value: precision }); + + callback(null, stat.mtime, precision); + }); + }); +} + +function getMtime(precision) { + let now = Date.now(); + + if (precision === 's') { + now = Math.ceil(now / 1000) * 1000; + } + + return new Date(now); +} + +module.exports.probe = probe; +module.exports.getMtime = getMtime; diff --git a/server/libs/retry/index.js b/server/libs/retry/index.js new file mode 100644 index 00000000..dcb57680 --- /dev/null +++ b/server/libs/retry/index.js @@ -0,0 +1,100 @@ +var RetryOperation = require('./retry_operation'); + +exports.operation = function(options) { + var timeouts = exports.timeouts(options); + return new RetryOperation(timeouts, { + forever: options && options.forever, + unref: options && options.unref, + maxRetryTime: options && options.maxRetryTime + }); +}; + +exports.timeouts = function(options) { + if (options instanceof Array) { + return [].concat(options); + } + + var opts = { + retries: 10, + factor: 2, + minTimeout: 1 * 1000, + maxTimeout: Infinity, + randomize: false + }; + for (var key in options) { + opts[key] = options[key]; + } + + if (opts.minTimeout > opts.maxTimeout) { + throw new Error('minTimeout is greater than maxTimeout'); + } + + var timeouts = []; + for (var i = 0; i < opts.retries; i++) { + timeouts.push(this.createTimeout(i, opts)); + } + + if (options && options.forever && !timeouts.length) { + timeouts.push(this.createTimeout(i, opts)); + } + + // sort the array numerically ascending + timeouts.sort(function(a,b) { + return a - b; + }); + + return timeouts; +}; + +exports.createTimeout = function(attempt, opts) { + var random = (opts.randomize) + ? (Math.random() + 1) + : 1; + + var timeout = Math.round(random * opts.minTimeout * Math.pow(opts.factor, attempt)); + timeout = Math.min(timeout, opts.maxTimeout); + + return timeout; +}; + +exports.wrap = function(obj, options, methods) { + if (options instanceof Array) { + methods = options; + options = null; + } + + if (!methods) { + methods = []; + for (var key in obj) { + if (typeof obj[key] === 'function') { + methods.push(key); + } + } + } + + for (var i = 0; i < methods.length; i++) { + var method = methods[i]; + var original = obj[method]; + + obj[method] = function retryWrapper(original) { + var op = exports.operation(options); + var args = Array.prototype.slice.call(arguments, 1); + var callback = args.pop(); + + args.push(function(err) { + if (op.retry(err)) { + return; + } + if (err) { + arguments[0] = op.mainError(); + } + callback.apply(this, arguments); + }); + + op.attempt(function() { + original.apply(obj, args); + }); + }.bind(obj, original); + obj[method].options = options; + } +}; diff --git a/server/libs/retry/retry_operation.js b/server/libs/retry/retry_operation.js new file mode 100644 index 00000000..1e564696 --- /dev/null +++ b/server/libs/retry/retry_operation.js @@ -0,0 +1,158 @@ +function RetryOperation(timeouts, options) { + // Compatibility for the old (timeouts, retryForever) signature + if (typeof options === 'boolean') { + options = { forever: options }; + } + + this._originalTimeouts = JSON.parse(JSON.stringify(timeouts)); + this._timeouts = timeouts; + this._options = options || {}; + this._maxRetryTime = options && options.maxRetryTime || Infinity; + this._fn = null; + this._errors = []; + this._attempts = 1; + this._operationTimeout = null; + this._operationTimeoutCb = null; + this._timeout = null; + this._operationStart = null; + + if (this._options.forever) { + this._cachedTimeouts = this._timeouts.slice(0); + } +} +module.exports = RetryOperation; + +RetryOperation.prototype.reset = function() { + this._attempts = 1; + this._timeouts = this._originalTimeouts; +} + +RetryOperation.prototype.stop = function() { + if (this._timeout) { + clearTimeout(this._timeout); + } + + this._timeouts = []; + this._cachedTimeouts = null; +}; + +RetryOperation.prototype.retry = function(err) { + if (this._timeout) { + clearTimeout(this._timeout); + } + + if (!err) { + return false; + } + var currentTime = new Date().getTime(); + if (err && currentTime - this._operationStart >= this._maxRetryTime) { + this._errors.unshift(new Error('RetryOperation timeout occurred')); + return false; + } + + this._errors.push(err); + + var timeout = this._timeouts.shift(); + if (timeout === undefined) { + if (this._cachedTimeouts) { + // retry forever, only keep last error + this._errors.splice(this._errors.length - 1, this._errors.length); + this._timeouts = this._cachedTimeouts.slice(0); + timeout = this._timeouts.shift(); + } else { + return false; + } + } + + var self = this; + var timer = setTimeout(function() { + self._attempts++; + + if (self._operationTimeoutCb) { + self._timeout = setTimeout(function() { + self._operationTimeoutCb(self._attempts); + }, self._operationTimeout); + + if (self._options.unref) { + self._timeout.unref(); + } + } + + self._fn(self._attempts); + }, timeout); + + if (this._options.unref) { + timer.unref(); + } + + return true; +}; + +RetryOperation.prototype.attempt = function(fn, timeoutOps) { + this._fn = fn; + + if (timeoutOps) { + if (timeoutOps.timeout) { + this._operationTimeout = timeoutOps.timeout; + } + if (timeoutOps.cb) { + this._operationTimeoutCb = timeoutOps.cb; + } + } + + var self = this; + if (this._operationTimeoutCb) { + this._timeout = setTimeout(function() { + self._operationTimeoutCb(); + }, self._operationTimeout); + } + + this._operationStart = new Date().getTime(); + + this._fn(this._attempts); +}; + +RetryOperation.prototype.try = function(fn) { + console.log('Using RetryOperation.try() is deprecated'); + this.attempt(fn); +}; + +RetryOperation.prototype.start = function(fn) { + console.log('Using RetryOperation.start() is deprecated'); + this.attempt(fn); +}; + +RetryOperation.prototype.start = RetryOperation.prototype.try; + +RetryOperation.prototype.errors = function() { + return this._errors; +}; + +RetryOperation.prototype.attempts = function() { + return this._attempts; +}; + +RetryOperation.prototype.mainError = function() { + if (this._errors.length === 0) { + return null; + } + + var counts = {}; + var mainError = null; + var mainErrorCount = 0; + + for (var i = 0; i < this._errors.length; i++) { + var error = this._errors[i]; + var message = error.message; + var count = (counts[message] || 0) + 1; + + counts[message] = count; + + if (count >= mainErrorCount) { + mainError = error; + mainErrorCount = count; + } + } + + return mainError; +}; diff --git a/server/libs/signalExit/index.js b/server/libs/signalExit/index.js new file mode 100644 index 00000000..93703f36 --- /dev/null +++ b/server/libs/signalExit/index.js @@ -0,0 +1,202 @@ +// Note: since nyc uses this module to output coverage, any lines +// that are in the direct sync flow of nyc's outputCoverage are +// ignored, since we can never get coverage for them. +// grab a reference to node's real process object right away +var process = global.process + +const processOk = function (process) { + return process && + typeof process === 'object' && + typeof process.removeListener === 'function' && + typeof process.emit === 'function' && + typeof process.reallyExit === 'function' && + typeof process.listeners === 'function' && + typeof process.kill === 'function' && + typeof process.pid === 'number' && + typeof process.on === 'function' +} + +// some kind of non-node environment, just no-op +/* istanbul ignore if */ +if (!processOk(process)) { + module.exports = function () { + return function () {} + } +} else { + var assert = require('assert') + var signals = require('./signals.js') + var isWin = /^win/i.test(process.platform) + + var EE = require('events') + /* istanbul ignore if */ + if (typeof EE !== 'function') { + EE = EE.EventEmitter + } + + var emitter + if (process.__signal_exit_emitter__) { + emitter = process.__signal_exit_emitter__ + } else { + emitter = process.__signal_exit_emitter__ = new EE() + emitter.count = 0 + emitter.emitted = {} + } + + // Because this emitter is a global, we have to check to see if a + // previous version of this library failed to enable infinite listeners. + // I know what you're about to say. But literally everything about + // signal-exit is a compromise with evil. Get used to it. + if (!emitter.infinite) { + emitter.setMaxListeners(Infinity) + emitter.infinite = true + } + + module.exports = function (cb, opts) { + /* istanbul ignore if */ + if (!processOk(global.process)) { + return function () {} + } + assert.equal(typeof cb, 'function', 'a callback must be provided for exit handler') + + if (loaded === false) { + load() + } + + var ev = 'exit' + if (opts && opts.alwaysLast) { + ev = 'afterexit' + } + + var remove = function () { + emitter.removeListener(ev, cb) + if (emitter.listeners('exit').length === 0 && + emitter.listeners('afterexit').length === 0) { + unload() + } + } + emitter.on(ev, cb) + + return remove + } + + var unload = function unload () { + if (!loaded || !processOk(global.process)) { + return + } + loaded = false + + signals.forEach(function (sig) { + try { + process.removeListener(sig, sigListeners[sig]) + } catch (er) {} + }) + process.emit = originalProcessEmit + process.reallyExit = originalProcessReallyExit + emitter.count -= 1 + } + module.exports.unload = unload + + var emit = function emit (event, code, signal) { + /* istanbul ignore if */ + if (emitter.emitted[event]) { + return + } + emitter.emitted[event] = true + emitter.emit(event, code, signal) + } + + // { : , ... } + var sigListeners = {} + signals.forEach(function (sig) { + sigListeners[sig] = function listener () { + /* istanbul ignore if */ + if (!processOk(global.process)) { + return + } + // If there are no other listeners, an exit is coming! + // Simplest way: remove us and then re-send the signal. + // We know that this will kill the process, so we can + // safely emit now. + var listeners = process.listeners(sig) + if (listeners.length === emitter.count) { + unload() + emit('exit', null, sig) + /* istanbul ignore next */ + emit('afterexit', null, sig) + /* istanbul ignore next */ + if (isWin && sig === 'SIGHUP') { + // "SIGHUP" throws an `ENOSYS` error on Windows, + // so use a supported signal instead + sig = 'SIGINT' + } + /* istanbul ignore next */ + process.kill(process.pid, sig) + } + } + }) + + module.exports.signals = function () { + return signals + } + + var loaded = false + + var load = function load () { + if (loaded || !processOk(global.process)) { + return + } + loaded = true + + // This is the number of onSignalExit's that are in play. + // It's important so that we can count the correct number of + // listeners on signals, and don't wait for the other one to + // handle it instead of us. + emitter.count += 1 + + signals = signals.filter(function (sig) { + try { + process.on(sig, sigListeners[sig]) + return true + } catch (er) { + return false + } + }) + + process.emit = processEmit + process.reallyExit = processReallyExit + } + module.exports.load = load + + var originalProcessReallyExit = process.reallyExit + var processReallyExit = function processReallyExit (code) { + /* istanbul ignore if */ + if (!processOk(global.process)) { + return + } + process.exitCode = code || /* istanbul ignore next */ 0 + emit('exit', process.exitCode, null) + /* istanbul ignore next */ + emit('afterexit', process.exitCode, null) + /* istanbul ignore next */ + originalProcessReallyExit.call(process, process.exitCode) + } + + var originalProcessEmit = process.emit + var processEmit = function processEmit (ev, arg) { + if (ev === 'exit' && processOk(global.process)) { + /* istanbul ignore else */ + if (arg !== undefined) { + process.exitCode = arg + } + var ret = originalProcessEmit.apply(this, arguments) + /* istanbul ignore next */ + emit('exit', process.exitCode, null) + /* istanbul ignore next */ + emit('afterexit', process.exitCode, null) + /* istanbul ignore next */ + return ret + } else { + return originalProcessEmit.apply(this, arguments) + } + } +} diff --git a/server/libs/signalExit/signals.js b/server/libs/signalExit/signals.js new file mode 100644 index 00000000..3bd67a8a --- /dev/null +++ b/server/libs/signalExit/signals.js @@ -0,0 +1,53 @@ +// This is not the set of all possible signals. +// +// It IS, however, the set of all signals that trigger +// an exit on either Linux or BSD systems. Linux is a +// superset of the signal names supported on BSD, and +// the unknown signals just fail to register, so we can +// catch that easily enough. +// +// Don't bother with SIGKILL. It's uncatchable, which +// means that we can't fire any callbacks anyway. +// +// If a user does happen to register a handler on a non- +// fatal signal like SIGWINCH or something, and then +// exit, it'll end up firing `process.emit('exit')`, so +// the handler will be fired anyway. +// +// SIGBUS, SIGFPE, SIGSEGV and SIGILL, when not raised +// artificially, inherently leave the process in a +// state from which it is not safe to try and enter JS +// listeners. +module.exports = [ + 'SIGABRT', + 'SIGALRM', + 'SIGHUP', + 'SIGINT', + 'SIGTERM' +] + +if (process.platform !== 'win32') { + module.exports.push( + 'SIGVTALRM', + 'SIGXCPU', + 'SIGXFSZ', + 'SIGUSR2', + 'SIGTRAP', + 'SIGSYS', + 'SIGQUIT', + 'SIGIOT' + // should detect profiler and enable/disable accordingly. + // see #21 + // 'SIGPROF' + ) +} + +if (process.platform === 'linux') { + module.exports.push( + 'SIGIO', + 'SIGPOLL', + 'SIGPWR', + 'SIGSTKFLT', + 'SIGUNUSED' + ) +} diff --git a/server/utils/dbMigration.js b/server/utils/dbMigration.js index 0a7b86a6..5c0388e0 100644 --- a/server/utils/dbMigration.js +++ b/server/utils/dbMigration.js @@ -1,6 +1,6 @@ const Path = require('path') const fs = require('fs-extra') -const njodb = require('../njodb') +const njodb = require('../libs/njodb') const { SupportedEbookTypes } = require('./globals') const { PlayMethod } = require('./constants')