diff --git a/.gitignore b/.gitignore index 9360600a..0690f38f 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,8 @@ /deploy/ /coverage/ /.nyc_output/ +/ffmpeg* +/ffprobe* sw.* .DS_STORE diff --git a/server/Server.js b/server/Server.js index 3aed50e0..9d2e3996 100644 --- a/server/Server.js +++ b/server/Server.js @@ -33,6 +33,7 @@ const AudioMetadataMangaer = require('./managers/AudioMetadataManager') const RssFeedManager = require('./managers/RssFeedManager') const CronManager = require('./managers/CronManager') const ApiCacheManager = require('./managers/ApiCacheManager') +const BinaryManager = require('./managers/BinaryManager') const LibraryScanner = require('./scanner/LibraryScanner') //Import the main Passport and Express-Session library @@ -74,6 +75,7 @@ class Server { this.rssFeedManager = new RssFeedManager() this.cronManager = new CronManager(this.podcastManager) this.apiCacheManager = new ApiCacheManager() + this.binaryManager = new BinaryManager() // Routers this.apiRouter = new ApiRouter(this) @@ -120,6 +122,11 @@ class Server { await this.cronManager.init(libraries) this.apiCacheManager.init() + // Download ffmpeg & ffprobe if not found (Currently only in use for Windows installs) + if (global.isWin || Logger.isDev) { + await this.binaryManager.init() + } + if (Database.serverSettings.scannerDisableWatcher) { Logger.info(`[Server] Watcher is disabled`) this.watcher.disabled = true diff --git a/server/libs/ffbinaries/index.js b/server/libs/ffbinaries/index.js new file mode 100644 index 00000000..b0660f08 --- /dev/null +++ b/server/libs/ffbinaries/index.js @@ -0,0 +1,315 @@ +const os = require('os') +const path = require('path') +const axios = require('axios') +const fse = require('../fsExtra') +const async = require('../async') +const StreamZip = require('../nodeStreamZip') +const { finished } = require('stream/promises') + +var API_URL = 'https://ffbinaries.com/api/v1' + +var RUNTIME_CACHE = {} +var errorMsgs = { + connectionIssues: 'Couldn\'t connect to ffbinaries.com API. Check your Internet connection.', + parsingVersionData: 'Couldn\'t parse retrieved version data.', + parsingVersionList: 'Couldn\'t parse the list of available versions.', + notFound: 'Requested data not found.', + incorrectVersionParam: '"version" parameter must be a string.' +} + +function ensureDirSync(dir) { + try { + fse.accessSync(dir) + } catch (e) { + fse.mkdirSync(dir) + } +} + +/** + * Resolves the platform key based on input string + */ +function resolvePlatform(input) { + var rtn = null + + switch (input) { + case 'mac': + case 'osx': + case 'mac-64': + case 'osx-64': + rtn = 'osx-64' + break + + case 'linux': + case 'linux-32': + rtn = 'linux-32' + break + + case 'linux-64': + rtn = 'linux-64' + break + + case 'linux-arm': + case 'linux-armel': + rtn = 'linux-armel' + break + + case 'linux-armhf': + rtn = 'linux-armhf' + break + + case 'win': + case 'win-32': + case 'windows': + case 'windows-32': + rtn = 'windows-32' + break + + case 'win-64': + case 'windows-64': + rtn = 'windows-64' + break + + default: + rtn = null + } + + return rtn +} +/** + * Detects the platform of the machine the script is executed on. + * Object can be provided to detect platform from info derived elsewhere. + * + * @param {object} osinfo Contains "type" and "arch" properties + */ +function detectPlatform(osinfo) { + var inputIsValid = typeof osinfo === 'object' && typeof osinfo.type === 'string' && typeof osinfo.arch === 'string' + var type = (inputIsValid ? osinfo.type : os.type()).toLowerCase() + var arch = (inputIsValid ? osinfo.arch : os.arch()).toLowerCase() + + if (type === 'darwin') { + return 'osx-64' + } + + if (type === 'windows_nt') { + return arch === 'x64' ? 'windows-64' : 'windows-32' + } + + if (type === 'linux') { + if (arch === 'arm' || arch === 'arm64') { + return 'linux-armel' + } + return arch === 'x64' ? 'linux-64' : 'linux-32' + } + + return null +} +/** + * Gets the binary filename (appends exe in Windows) + * + * @param {string} component "ffmpeg", "ffplay", "ffprobe" or "ffserver" + * @param {platform} platform "ffmpeg", "ffplay", "ffprobe" or "ffserver" + */ +function getBinaryFilename(component, platform) { + var platformCode = resolvePlatform(platform) + if (platformCode === 'windows-32' || platformCode === 'windows-64') { + return component + '.exe' + } + return component +} + +function listPlatforms() { + return ['osx-64', 'linux-32', 'linux-64', 'linux-armel', 'linux-armhf', 'windows-32', 'windows-64'] +} + +/** + * + * @returns {Promise} array of version strings + */ +function listVersions() { + if (RUNTIME_CACHE.versionsAll) { + return RUNTIME_CACHE.versionsAll + } + return axios.get(API_URL).then((res) => { + if (!res.data?.versions || !Object.keys(res.data.versions)?.length) { + throw new Error(errorMsgs.parsingVersionList) + } + const versionKeys = Object.keys(res.data.versions) + RUNTIME_CACHE.versionsAll = versionKeys + return versionKeys + }) +} +/** + * Gets full data set from ffbinaries.com + */ +function getVersionData(version) { + if (RUNTIME_CACHE[version]) { + return RUNTIME_CACHE[version] + } + + if (version && typeof version !== 'string') { + throw new Error(errorMsgs.incorrectVersionParam) + } + + var url = version ? '/version/' + version : '/latest' + + return axios.get(`${API_URL}${url}`).then((res) => { + RUNTIME_CACHE[version] = res.data + return res.data + }).catch((error) => { + if (error.response?.status == 404) { + throw new Error(errorMsgs.notFound) + } else { + throw new Error(errorMsgs.connectionIssues) + } + }) +} + +/** + * Download file(s) and save them in the specified directory + */ +async function downloadUrls(components, urls, opts) { + const destinationDir = opts.destination + const results = [] + const remappedUrls = [] + + if (components && !Array.isArray(components)) { + components = [components] + } else if (!components || !Array.isArray(components)) { + components = [] + } + + // returns an array of objects like this: {component: 'ffmpeg', url: 'https://...'} + if (typeof urls === 'object') { + for (const key in urls) { + if (components.includes(key) && urls[key]) { + remappedUrls.push({ + component: key, + url: urls[key] + }) + } + } + } + + + async function extractZipToDestination(zipFilename) { + const oldpath = path.join(destinationDir, zipFilename) + const zip = new StreamZip.async({ file: oldpath }) + const count = await zip.extract(null, destinationDir) + await zip.close() + } + + + await async.each(remappedUrls, async function (urlObject) { + try { + const url = urlObject.url + + const zipFilename = url.split('/').pop() + const binFilenameBase = urlObject.component + const binFilename = getBinaryFilename(binFilenameBase, opts.platform || detectPlatform()) + + let runningTotal = 0 + let totalFilesize + let interval + + + if (typeof opts.tickerFn === 'function') { + opts.tickerInterval = parseInt(opts.tickerInterval, 10) + const tickerInterval = (!Number.isNaN(opts.tickerInterval)) ? opts.tickerInterval : 1000 + const tickData = { filename: zipFilename, progress: 0 } + + // Schedule next ticks + interval = setInterval(function () { + if (totalFilesize && runningTotal == totalFilesize) { + return clearInterval(interval) + } + tickData.progress = totalFilesize > -1 ? runningTotal / totalFilesize : 0 + + opts.tickerFn(tickData) + }, tickerInterval) + } + + + // Check if file already exists in target directory + const binPath = path.join(destinationDir, binFilename) + if (!opts.force && await fse.pathExists(binPath)) { + // if the accessSync method doesn't throw we know the binary already exists + results.push({ + filename: binFilename, + path: destinationDir, + status: 'File exists', + code: 'FILE_EXISTS' + }) + clearInterval(interval) + return + } + + if (opts.quiet) clearInterval(interval) + + const zipPath = path.join(destinationDir, zipFilename) + const zipFileTempName = zipPath + '.part' + const zipFileFinalName = zipPath + + const response = await axios({ + url, + method: 'GET', + responseType: 'stream' + }) + totalFilesize = response.headers?.['content-length'] || [] + + const writer = fse.createWriteStream(zipFileTempName) + response.data.on('data', (chunk) => { + runningTotal += chunk.length + }) + response.data.pipe(writer) + await finished(writer) + await fse.rename(zipFileTempName, zipFileFinalName) + await extractZipToDestination(zipFilename) + await fse.remove(zipFileFinalName) + + results.push({ + filename: binFilename, + path: destinationDir, + size: Math.floor(totalFilesize / 1024 / 1024 * 1000) / 1000 + 'MB', + status: 'File extracted to destination (downloaded from "' + url + '")', + code: 'DONE_CLEAN' + }) + } catch (err) { + console.error(`Failed to download or extract file for component: ${urlObject.component}`, err) + } + }) + + return results +} + +/** + * Gets binaries for the platform + * It will get the data from ffbinaries, pick the correct files + * and save it to the specified directory + * + * @param {Array} components + * @param {Object} [opts] + */ +async function downloadBinaries(components, opts = {}) { + var platform = resolvePlatform(opts.platform) || detectPlatform() + + opts.destination = path.resolve(opts.destination || '.') + ensureDirSync(opts.destination) + + const versionData = await getVersionData(opts.version) + const urls = versionData?.bin?.[platform] + if (!urls) { + throw new Error('No URLs!') + } + + return await downloadUrls(components, urls, opts) +} + +module.exports = { + downloadBinaries: downloadBinaries, + getVersionData: getVersionData, + listVersions: listVersions, + listPlatforms: listPlatforms, + detectPlatform: detectPlatform, + resolvePlatform: resolvePlatform, + getBinaryFilename: getBinaryFilename +} \ No newline at end of file diff --git a/server/managers/BinaryManager.js b/server/managers/BinaryManager.js new file mode 100644 index 00000000..e9aab609 --- /dev/null +++ b/server/managers/BinaryManager.js @@ -0,0 +1,74 @@ +const path = require('path') +const which = require('../libs/which') +const fs = require('../libs/fsExtra') +const ffbinaries = require('../libs/ffbinaries') +const Logger = require('../Logger') +const fileUtils = require('../utils/fileUtils') + +class BinaryManager { + + defaultRequiredBinaries = [ + { name: 'ffmpeg', envVariable: 'FFMPEG_PATH' }, + { name: 'ffprobe', envVariable: 'FFPROBE_PATH' } + ] + + constructor(requiredBinaries = this.defaultRequiredBinaries) { + this.requiredBinaries = requiredBinaries + this.mainInstallPath = process.pkg ? path.dirname(process.execPath) : global.appRoot + this.altInstallPath = global.ConfigPath + } + + async init() { + if (this.initialized) return + const missingBinaries = await this.findRequiredBinaries() + if (missingBinaries.length == 0) return + await this.install(missingBinaries) + const missingBinariesAfterInstall = await this.findRequiredBinaries() + if (missingBinariesAfterInstall.length != 0) { + Logger.error(`[BinaryManager] Failed to find or install required binaries: ${missingBinariesAfterInstall.join(', ')}`) + process.exit(1) + } + this.initialized = true + } + + async findRequiredBinaries() { + const missingBinaries = [] + for (const binary of this.requiredBinaries) { + const binaryPath = await this.findBinary(binary.name, binary.envVariable) + if (binaryPath) { + Logger.info(`[BinaryManager] Found ${binary.name} at ${binaryPath}`) + if (process.env[binary.envVariable] !== binaryPath) { + Logger.info(`[BinaryManager] Updating process.env.${binary.envVariable}`) + process.env[binary.envVariable] = binaryPath + } + } else { + Logger.info(`[BinaryManager] ${binary.name} not found`) + missingBinaries.push(binary.name) + } + } + return missingBinaries + } + + async findBinary(name, envVariable) { + const executable = name + (process.platform == 'win32' ? '.exe' : '') + const defaultPath = process.env[envVariable] + if (defaultPath && await fs.pathExists(defaultPath)) return defaultPath + const whichPath = which.sync(executable, { nothrow: true }) + if (whichPath) return whichPath + const mainInstallPath = path.join(this.mainInstallPath, executable) + if (await fs.pathExists(mainInstallPath)) return mainInstallPath + const altInstallPath = path.join(this.altInstallPath, executable) + if (await fs.pathExists(altInstallPath)) return altInstallPath + return null + } + + async install(binaries) { + if (binaries.length == 0) return + Logger.info(`[BinaryManager] Installing binaries: ${binaries.join(', ')}`) + let destination = await fileUtils.isWritable(this.mainInstallPath) ? this.mainInstallPath : this.altInstallPath + await ffbinaries.downloadBinaries(binaries, { destination }) + Logger.info(`[BinaryManager] Binaries installed to ${destination}`) + } +} + +module.exports = BinaryManager \ No newline at end of file diff --git a/server/utils/fileUtils.js b/server/utils/fileUtils.js index 7ef5320d..89ad9e60 100644 --- a/server/utils/fileUtils.js +++ b/server/utils/fileUtils.js @@ -359,3 +359,22 @@ module.exports.encodeUriPath = (path) => { const uri = new URL(path, "file://") return uri.pathname } + +/** + * Check if directory is writable. + * This method is necessary because fs.access(directory, fs.constants.W_OK) does not work on Windows + * + * @param {string} directory + * @returns {boolean} + */ +module.exports.isWritable = async (directory) => { + try { + const accessTestFile = path.join(directory, 'accessTest') + await fs.writeFile(accessTestFile, '') + await fs.remove(accessTestFile) + return true + } catch (err) { + return false + } +} + diff --git a/test/server/managers/BinaryManager.test.js b/test/server/managers/BinaryManager.test.js new file mode 100644 index 00000000..b93973e0 --- /dev/null +++ b/test/server/managers/BinaryManager.test.js @@ -0,0 +1,264 @@ +const chai = require('chai') +const sinon = require('sinon') +const fs = require('../../../server/libs/fsExtra') +const fileUtils = require('../../../server/utils/fileUtils') +const which = require('../../../server/libs/which') +const ffbinaries = require('../../../server/libs/ffbinaries') +const path = require('path') +const BinaryManager = require('../../../server/managers/BinaryManager') + +const expect = chai.expect + +describe('BinaryManager', () => { + let binaryManager + + describe('init', () => { + let findStub + let installStub + let errorStub + let exitStub + + beforeEach(() => { + binaryManager = new BinaryManager() + findStub = sinon.stub(binaryManager, 'findRequiredBinaries') + installStub = sinon.stub(binaryManager, 'install') + errorStub = sinon.stub(console, 'error') + exitStub = sinon.stub(process, 'exit') + }) + + afterEach(() => { + findStub.restore() + installStub.restore() + errorStub.restore() + exitStub.restore() + }) + + it('should not install binaries if they are already found', async () => { + findStub.resolves([]) + + await binaryManager.init() + + expect(installStub.called).to.be.false + expect(findStub.calledOnce).to.be.true + expect(errorStub.called).to.be.false + expect(exitStub.called).to.be.false + }) + + it('should install missing binaries', async () => { + const missingBinaries = ['ffmpeg', 'ffprobe'] + const missingBinariesAfterInstall = [] + findStub.onFirstCall().resolves(missingBinaries) + findStub.onSecondCall().resolves(missingBinariesAfterInstall) + + await binaryManager.init() + + expect(findStub.calledTwice).to.be.true + expect(installStub.calledOnce).to.be.true + expect(errorStub.called).to.be.false + expect(exitStub.called).to.be.false + }) + + it('exit if binaries are not found after installation', async () => { + const missingBinaries = ['ffmpeg', 'ffprobe'] + const missingBinariesAfterInstall = ['ffmpeg', 'ffprobe'] + findStub.onFirstCall().resolves(missingBinaries) + findStub.onSecondCall().resolves(missingBinariesAfterInstall) + + await binaryManager.init() + + expect(findStub.calledTwice).to.be.true + expect(installStub.calledOnce).to.be.true + expect(errorStub.calledOnce).to.be.true + expect(exitStub.calledOnce).to.be.true + expect(exitStub.calledWith(1)).to.be.true + }) + }) + + + describe('findRequiredBinaries', () => { + let findBinaryStub + + beforeEach(() => { + const requiredBinaries = [{ name: 'ffmpeg', envVariable: 'FFMPEG_PATH' }] + binaryManager = new BinaryManager(requiredBinaries) + findBinaryStub = sinon.stub(binaryManager, 'findBinary') + }) + + afterEach(() => { + findBinaryStub.restore() + }) + + it('should put found paths in the correct environment variables', async () => { + const pathToFFmpeg = '/path/to/ffmpeg' + const missingBinaries = [] + delete process.env.FFMPEG_PATH + findBinaryStub.resolves(pathToFFmpeg) + + const result = await binaryManager.findRequiredBinaries() + + expect(result).to.deep.equal(missingBinaries) + expect(findBinaryStub.calledOnce).to.be.true + expect(process.env.FFMPEG_PATH).to.equal(pathToFFmpeg) + }) + + it('should add missing binaries to result', async () => { + const missingBinaries = ['ffmpeg'] + delete process.env.FFMPEG_PATH + findBinaryStub.resolves(null) + + const result = await binaryManager.findRequiredBinaries() + + expect(result).to.deep.equal(missingBinaries) + expect(findBinaryStub.calledOnce).to.be.true + expect(process.env.FFMPEG_PATH).to.be.undefined + }) + }) + + describe('install', () => { + let isWritableStub + let downloadBinariesStub + + beforeEach(() => { + binaryManager = new BinaryManager() + isWritableStub = sinon.stub(fileUtils, 'isWritable') + downloadBinariesStub = sinon.stub(ffbinaries, 'downloadBinaries') + binaryManager.mainInstallPath = '/path/to/main/install' + binaryManager.altInstallPath = '/path/to/alt/install' + }) + + afterEach(() => { + isWritableStub.restore() + downloadBinariesStub.restore() + }) + + it('should not install binaries if no binaries are passed', async () => { + const binaries = [] + + await binaryManager.install(binaries) + + expect(isWritableStub.called).to.be.false + expect(downloadBinariesStub.called).to.be.false + }) + + it('should install binaries in main install path if has access', async () => { + const binaries = ['ffmpeg'] + const destination = binaryManager.mainInstallPath + isWritableStub.withArgs(destination).resolves(true) + downloadBinariesStub.resolves() + + await binaryManager.install(binaries) + + expect(isWritableStub.calledOnce).to.be.true + expect(downloadBinariesStub.calledOnce).to.be.true + expect(downloadBinariesStub.calledWith(binaries, sinon.match({ destination: destination }))).to.be.true + }) + + it('should install binaries in alt install path if has no access to main', async () => { + const binaries = ['ffmpeg'] + const mainDestination = binaryManager.mainInstallPath + const destination = binaryManager.altInstallPath + isWritableStub.withArgs(mainDestination).resolves(false) + downloadBinariesStub.resolves() + + await binaryManager.install(binaries) + + expect(isWritableStub.calledOnce).to.be.true + expect(downloadBinariesStub.calledOnce).to.be.true + expect(downloadBinariesStub.calledWith(binaries, sinon.match({ destination: destination }))).to.be.true + }) + }) +}) + +describe('findBinary', () => { + let binaryManager + let fsPathExistsStub + let whichSyncStub + let mainInstallPath + let altInstallPath + + const name = 'ffmpeg' + const envVariable = 'FFMPEG_PATH' + const defaultPath = '/path/to/ffmpeg' + const executable = name + (process.platform == 'win32' ? '.exe' : '') + const whichPath = '/usr/bin/ffmpeg' + + + beforeEach(() => { + binaryManager = new BinaryManager() + fsPathExistsStub = sinon.stub(fs, 'pathExists') + whichSyncStub = sinon.stub(which, 'sync') + binaryManager.mainInstallPath = '/path/to/main/install' + mainInstallPath = path.join(binaryManager.mainInstallPath, executable) + binaryManager.altInstallPath = '/path/to/alt/install' + altInstallPath = path.join(binaryManager.altInstallPath, executable) + }) + + afterEach(() => { + fsPathExistsStub.restore() + whichSyncStub.restore() + }) + + it('should return defaultPath if it exists', async () => { + process.env[envVariable] = defaultPath + fsPathExistsStub.withArgs(defaultPath).resolves(true) + + const result = await binaryManager.findBinary(name, envVariable) + + expect(result).to.equal(defaultPath) + expect(fsPathExistsStub.calledOnceWith(defaultPath)).to.be.true + expect(whichSyncStub.notCalled).to.be.true + }) + + it('should return whichPath if it exists', async () => { + delete process.env[envVariable] + whichSyncStub.returns(whichPath) + + const result = await binaryManager.findBinary(name, envVariable) + + expect(result).to.equal(whichPath) + expect(fsPathExistsStub.notCalled).to.be.true + expect(whichSyncStub.calledOnce).to.be.true + }) + + it('should return mainInstallPath if it exists', async () => { + delete process.env[envVariable] + whichSyncStub.returns(null) + fsPathExistsStub.withArgs(mainInstallPath).resolves(true) + + const result = await binaryManager.findBinary(name, envVariable) + + expect(result).to.equal(mainInstallPath) + expect(whichSyncStub.calledOnce).to.be.true + expect(fsPathExistsStub.calledOnceWith(mainInstallPath)).to.be.true + }) + + it('should return altInstallPath if it exists', async () => { + delete process.env[envVariable] + whichSyncStub.returns(null) + fsPathExistsStub.withArgs(mainInstallPath).resolves(false) + fsPathExistsStub.withArgs(altInstallPath).resolves(true) + + const result = await binaryManager.findBinary(name, envVariable) + + expect(result).to.equal(altInstallPath) + expect(whichSyncStub.calledOnce).to.be.true + expect(fsPathExistsStub.calledTwice).to.be.true + expect(fsPathExistsStub.calledWith(mainInstallPath)).to.be.true + expect(fsPathExistsStub.calledWith(altInstallPath)).to.be.true + }) + + it('should return null if binary is not found', async () => { + delete process.env[envVariable] + whichSyncStub.returns(null) + fsPathExistsStub.withArgs(mainInstallPath).resolves(false) + fsPathExistsStub.withArgs(altInstallPath).resolves(false) + + const result = await binaryManager.findBinary(name, envVariable) + + expect(result).to.be.null + expect(whichSyncStub.calledOnce).to.be.true + expect(fsPathExistsStub.calledTwice).to.be.true + expect(fsPathExistsStub.calledWith(mainInstallPath)).to.be.true + expect(fsPathExistsStub.calledWith(altInstallPath)).to.be.true + }) +}) \ No newline at end of file