audiobookshelf/server/Server.js

428 lines
14 KiB
JavaScript
Raw Normal View History

2021-08-18 00:01:11 +02:00
const Path = require('path')
const Sequelize = require('sequelize')
2021-08-18 00:01:11 +02:00
const express = require('express')
const http = require('http')
const util = require('util')
2022-07-06 02:53:01 +02:00
const fs = require('./libs/fsExtra')
2022-07-07 02:10:25 +02:00
const fileUpload = require('./libs/expressFileupload')
2024-05-05 23:39:38 +02:00
const cookieParser = require('cookie-parser')
2021-08-18 00:01:11 +02:00
const { version } = require('../package.json')
// Utils
const fileUtils = require('./utils/fileUtils')
const Logger = require('./Logger')
2021-08-18 00:01:11 +02:00
const Auth = require('./Auth')
const Watcher = require('./Watcher')
2023-07-05 01:14:44 +02:00
const Database = require('./Database')
const SocketAuthority = require('./SocketAuthority')
const ApiRouter = require('./routers/ApiRouter')
const HlsRouter = require('./routers/HlsRouter')
const LogManager = require('./managers/LogManager')
2022-09-21 01:08:41 +02:00
const NotificationManager = require('./managers/NotificationManager')
const EmailManager = require('./managers/EmailManager')
const AbMergeManager = require('./managers/AbMergeManager')
const CacheManager = require('./managers/CacheManager')
2023-07-08 21:40:49 +02:00
const BackupManager = require('./managers/BackupManager')
const PlaybackSessionManager = require('./managers/PlaybackSessionManager')
const PodcastManager = require('./managers/PodcastManager')
const AudioMetadataMangaer = require('./managers/AudioMetadataManager')
const RssFeedManager = require('./managers/RssFeedManager')
const CronManager = require('./managers/CronManager')
const ApiCacheManager = require('./managers/ApiCacheManager')
2023-12-05 20:19:17 +01:00
const BinaryManager = require('./managers/BinaryManager')
const LibraryScanner = require('./scanner/LibraryScanner')
//Import the main Passport and Express-Session library
const passport = require('passport')
const expressSession = require('express-session')
2021-08-18 00:01:11 +02:00
class Server {
2024-03-11 17:11:13 +01:00
constructor(SOURCE, PORT, HOST, CONFIG_PATH, METADATA_PATH, ROUTER_BASE_PATH) {
2021-08-18 00:01:11 +02:00
this.Port = PORT
this.Host = HOST
global.Source = SOURCE
global.isWin = process.platform === 'win32'
global.ConfigPath = fileUtils.filePathToPOSIX(Path.normalize(CONFIG_PATH))
global.MetadataPath = fileUtils.filePathToPOSIX(Path.normalize(METADATA_PATH))
global.RouterBasePath = ROUTER_BASE_PATH
Implement X-Accel Redirect This patch implements [X-Accel](https://www.nginx.com/resources/wiki/start/topics/examples/x-accel/) redirect headers as an optional way for offloading static file delivery from Express to Nginx, which is far better optimized for static file delivery. This provides a really easy to configure way for getting a huge performance boost over delivering all files through Audiobookshelf. How it works ------------ The way this works is basically that Audiobookshelf gets an HTTP request for delivering a static file (let's say an audiobook). It will first check the user is authorized and then convert the API path to a local file path. Now, instead of reading and delivering the file, Audiobookshelf will return just the HTTP header with an additional `X-Accel-Redirect` pointing to the file location on the file syste. This header is picked up by Nginx which will then deliver the file. Configuration ------------- The configuration for this is very simple. You need to run Nginx as reverse proxy and it must have access to your Audiobookshelf data folder. You then configure Audiobookshelf to use X-Accel by setting `USE_X_ACCEL=/protected`. The path is the internal redirect path used by Nginx. In the Nginx configuration you then configure this location and map it to the storage area to serve like this: ``` location /protected/ { internal; alias /; } ``` That's all. Impact ------ I just did a very simple performance test, downloading a 1170620819 bytes large audiobook file from another machine on the same network like this, using `time -p` to measure how log the process took: ```sh URL='https://url to audiobook…' for i in `seq 1 50` do echo "$i" curl -s -o /dev/null "${URL}" done ``` This sequential test with 50 iterations and without x-accel resulted in: ``` real 413.42 user 197.11 sys 82.04 ``` That is an average download speed of about 1080 MBit/s. With X-Accel enabled, serving the files through Nginx, the same test yielded the following results: ``` real 200.37 user 86.95 sys 29.79 ``` That is an average download speed of about 2229 MBit/s, more than doubling the previous speed. I have also run the same test with 4 parallel processes and 25 downloads each. Without x-accel, that test resulted in: ``` real 364.89 user 273.09 sys 112.75 ``` That is an average speed of about 2448 MBit/s. With X-Accel enabled, the parallel test also shows a significant speedup: ``` real 167.19 user 195.62 sys 78.61 ``` That is an average speed of about 5342 MBit/s. While doing that, I also peaked at the system load which was a bit lower when using X-Accel. Even though the system was delivering far more data. But I just looked at the `load1` values and did not build a proper test for that. That means, I cant provide any definitive data. Supported Media --------------- The current implementation works for audio files and book covers. There are other media files which would benefit from this mechanism like feed covers or author pictures. But that's something for a future developer ;-)
2022-11-25 23:41:35 +01:00
global.XAccel = process.env.USE_X_ACCEL
if (!fs.pathExistsSync(global.ConfigPath)) {
fs.mkdirSync(global.ConfigPath)
}
if (!fs.pathExistsSync(global.MetadataPath)) {
fs.mkdirSync(global.MetadataPath)
}
2021-08-18 00:01:11 +02:00
this.watcher = new Watcher()
2023-07-05 01:14:44 +02:00
this.auth = new Auth()
// Managers
2023-07-05 01:14:44 +02:00
this.notificationManager = new NotificationManager()
this.emailManager = new EmailManager()
2023-07-08 21:40:49 +02:00
this.backupManager = new BackupManager()
2023-10-20 23:39:32 +02:00
this.abMergeManager = new AbMergeManager()
2023-07-05 01:14:44 +02:00
this.playbackSessionManager = new PlaybackSessionManager()
2023-10-20 23:39:32 +02:00
this.podcastManager = new PodcastManager(this.watcher, this.notificationManager)
this.audioMetadataManager = new AudioMetadataMangaer()
2023-07-05 01:14:44 +02:00
this.rssFeedManager = new RssFeedManager()
this.cronManager = new CronManager(this.podcastManager)
this.apiCacheManager = new ApiCacheManager()
2023-12-05 20:19:17 +01:00
this.binaryManager = new BinaryManager()
// Routers
this.apiRouter = new ApiRouter(this)
2023-07-05 01:14:44 +02:00
this.hlsRouter = new HlsRouter(this.auth, this.playbackSessionManager)
Logger.logManager = new LogManager()
2021-08-18 00:01:11 +02:00
this.server = null
this.io = null
}
authMiddleware(req, res, next) {
// ask passportjs if the current request is authenticated
this.auth.isAuthenticated(req, res, next)
}
cancelLibraryScan(libraryId) {
LibraryScanner.setCancelLibraryScan(libraryId)
}
/**
* Initialize database, backups, logs, rss feeds, cron jobs & watcher
* Cleanup stale/invalid data
*/
2021-08-18 00:01:11 +02:00
async init() {
Logger.info('[Server] Init v' + version)
await this.playbackSessionManager.removeOrphanStreams()
2021-09-23 03:40:35 +02:00
2023-07-05 01:14:44 +02:00
await Database.init(false)
await Logger.logManager.init()
2022-07-19 00:19:16 +02:00
// Create token secret if does not exist (Added v2.1.0)
2023-07-05 01:14:44 +02:00
if (!Database.serverSettings.tokenSecret) {
2022-07-19 00:19:16 +02:00
await this.auth.initTokenSecret()
}
await this.cleanUserData() // Remove invalid user item progress
2023-09-07 00:48:50 +02:00
await CacheManager.ensureCachePaths()
2022-06-18 20:11:15 +02:00
2023-07-08 21:40:49 +02:00
await this.backupManager.init()
await this.rssFeedManager.init()
2023-08-20 20:34:03 +02:00
const libraries = await Database.libraryModel.getAllOldLibraries()
await this.cronManager.init(libraries)
this.apiCacheManager.init()
// Download ffmpeg & ffprobe if not found (Currently only in use for Windows installs)
if (global.isWin || Logger.isDev) {
await this.binaryManager.init()
}
2023-07-05 01:14:44 +02:00
if (Database.serverSettings.scannerDisableWatcher) {
Logger.info(`[Server] Watcher is disabled`)
this.watcher.disabled = true
} else {
this.watcher.initWatcher(libraries)
}
2021-08-18 00:01:11 +02:00
}
/**
* Listen for SIGINT and uncaught exceptions
*/
initProcessEventListeners() {
let sigintAlreadyReceived = false
process.on('SIGINT', async () => {
if (!sigintAlreadyReceived) {
sigintAlreadyReceived = true
Logger.info('SIGINT (Ctrl+C) received. Shutting down...')
await this.stop()
Logger.info('Server stopped. Exiting.')
} else {
Logger.info('SIGINT (Ctrl+C) received again. Exiting immediately.')
}
process.exit(0)
})
/**
* @see https://nodejs.org/api/process.html#event-uncaughtexceptionmonitor
*/
process.on('uncaughtExceptionMonitor', async (error, origin) => {
await Logger.fatal(`[Server] Uncaught exception origin: ${origin}, error:`, util.format('%O', error))
})
/**
* @see https://nodejs.org/api/process.html#event-unhandledrejection
*/
process.on('unhandledRejection', async (reason, promise) => {
await Logger.fatal(`[Server] Unhandled rejection: ${reason}, promise:`, util.format('%O', promise))
process.exit(1)
})
}
2021-08-18 00:01:11 +02:00
async start() {
Logger.info('=== Starting Server ===')
this.initProcessEventListeners()
2021-08-18 00:01:11 +02:00
await this.init()
const app = express()
/**
* @temporary
* This is necessary for the ebook & cover API endpoint in the mobile apps
* The mobile app ereader is using fetch api in Capacitor that is currently difficult to switch to native requests
* so we have to allow cors for specific origins to the /api/items/:id/ebook endpoint
* The cover image is fetched with XMLHttpRequest in the mobile apps to load into a canvas and extract colors
* @see https://ionicframework.com/docs/troubleshooting/cors
2024-05-05 23:39:38 +02:00
*
* Running in development allows cors to allow testing the mobile apps in the browser
*/
app.use((req, res, next) => {
if (Logger.isDev || req.path.match(/\/api\/items\/([a-z0-9-]{36})\/(ebook|cover)(\/[0-9]+)?/)) {
const allowedOrigins = ['capacitor://localhost', 'http://localhost']
2024-05-05 23:39:38 +02:00
if (Logger.isDev || allowedOrigins.some((o) => o === req.get('origin'))) {
res.header('Access-Control-Allow-Origin', req.get('origin'))
2024-05-05 23:39:38 +02:00
res.header('Access-Control-Allow-Methods', 'GET, POST, PATCH, PUT, DELETE, OPTIONS')
res.header('Access-Control-Allow-Headers', '*')
res.header('Access-Control-Allow-Credentials', true)
if (req.method === 'OPTIONS') {
return res.sendStatus(200)
}
}
}
next()
})
// parse cookies in requests
app.use(cookieParser())
// enable express-session
2024-05-05 23:39:38 +02:00
app.use(
expressSession({
secret: global.ServerSettings.tokenSecret,
resave: false,
saveUninitialized: false,
cookie: {
// also send the cookie if were are not on https (not every use has https)
secure: false
}
})
)
// init passport.js
app.use(passport.initialize())
// register passport in express-session
app.use(passport.session())
// config passport.js
await this.auth.initPassportJs()
const router = express.Router()
app.use(global.RouterBasePath, router)
app.disable('x-powered-by')
2021-08-18 00:01:11 +02:00
this.server = http.createServer(app)
2024-05-05 23:39:38 +02:00
router.use(
fileUpload({
defCharset: 'utf8',
defParamCharset: 'utf8',
useTempFiles: true,
tempFileDir: Path.join(global.MetadataPath, 'tmp')
})
)
router.use(express.urlencoded({ extended: true, limit: '5mb' }))
router.use(express.json({ limit: '5mb' }))
2021-08-18 00:01:11 +02:00
// Static path to generated nuxt
const distPath = Path.join(global.appRoot, '/client/dist')
router.use(express.static(distPath))
// Static folder
router.use(express.static(Path.join(global.appRoot, 'static')))
2021-08-18 00:01:11 +02:00
router.use('/api', this.authMiddleware.bind(this), this.apiRouter.router)
router.use('/hls', this.authMiddleware.bind(this), this.hlsRouter.router)
// RSS Feed temp route
router.get('/feed/:slug', (req, res) => {
Logger.info(`[Server] Requesting rss feed ${req.params.slug}`)
this.rssFeedManager.getFeed(req, res)
})
router.get('/feed/:slug/cover*', (req, res) => {
this.rssFeedManager.getFeedCover(req, res)
})
router.get('/feed/:slug/item/:episodeId/*', (req, res) => {
Logger.debug(`[Server] Requesting rss feed episode ${req.params.slug}/${req.params.episodeId}`)
this.rssFeedManager.getFeedItem(req, res)
})
// Auth routes
await this.auth.initAuthRoutes(router)
// Client dynamic routes
const dyanimicRoutes = [
'/item/:id',
'/author/:id',
'/audiobook/:id/chapters',
'/audiobook/:id/edit',
'/audiobook/:id/manage',
'/library/:library',
'/library/:library/search',
'/library/:library/bookshelf/:id?',
'/library/:library/authors',
2023-11-28 23:39:52 +01:00
'/library/:library/narrators',
'/library/:library/series/:id?',
2022-09-17 22:23:33 +02:00
'/library/:library/podcast/search',
'/library/:library/podcast/latest',
'/library/:library/podcast/download-queue',
'/config/users/:id',
'/config/users/:id/sessions',
'/config/item-metadata-utils/:id',
2022-11-27 21:23:28 +01:00
'/collection/:id',
'/playlist/:id'
]
dyanimicRoutes.forEach((route) => router.get(route, (req, res) => res.sendFile(Path.join(distPath, 'index.html'))))
router.post('/init', (req, res) => {
2023-07-05 01:14:44 +02:00
if (Database.hasRootUser) {
Logger.error(`[Server] attempt to init server when server already has a root user`)
return res.sendStatus(500)
}
this.initializeServer(req, res)
})
router.get('/status', (req, res) => {
// status check for client to see if server has been initialized
// server has been initialized if a root user exists
const payload = {
app: 'audiobookshelf',
serverVersion: version,
2023-07-05 01:14:44 +02:00
isInit: Database.hasRootUser,
language: Database.serverSettings.language,
authMethods: Database.serverSettings.authActiveAuthMethods,
authFormData: Database.serverSettings.authFormData
}
if (!payload.isInit) {
payload.ConfigPath = global.ConfigPath
payload.MetadataPath = global.MetadataPath
}
res.json(payload)
})
router.get('/ping', (req, res) => {
2022-07-30 15:37:35 +02:00
Logger.info('Received ping')
2021-08-18 00:01:11 +02:00
res.json({ success: true })
})
app.get('/healthcheck', (req, res) => res.sendStatus(200))
2021-08-18 00:01:11 +02:00
this.server.listen(this.Port, this.Host, () => {
if (this.Host) Logger.info(`Listening on http://${this.Host}:${this.Port}`)
else Logger.info(`Listening on port :${this.Port}`)
2021-08-18 00:01:11 +02:00
})
// Start listening for socket connections
SocketAuthority.initialize(this)
2021-08-18 00:01:11 +02:00
}
async initializeServer(req, res) {
Logger.info(`[Server] Initializing new server`)
const newRoot = req.body.newRoot
const rootUsername = newRoot.username || 'root'
const rootPash = newRoot.password ? await this.auth.hashPass(newRoot.password) : ''
if (!rootPash) Logger.warn(`[Server] Creating root user with no password`)
await Database.createRootUser(rootUsername, rootPash, this.auth)
res.sendStatus(200)
}
/**
* Remove user media progress for items that no longer exist & remove seriesHideFrom that no longer exist
*/
async cleanUserData() {
// Get all media progress without an associated media item
2023-08-20 20:34:03 +02:00
const mediaProgressToRemove = await Database.mediaProgressModel.findAll({
where: {
'$podcastEpisode.id$': null,
'$book.id$': null
},
attributes: ['id'],
include: [
{
2023-08-20 20:34:03 +02:00
model: Database.bookModel,
attributes: ['id']
},
{
2023-08-20 20:34:03 +02:00
model: Database.podcastEpisodeModel,
attributes: ['id']
}
]
})
if (mediaProgressToRemove.length) {
// Remove media progress
2023-08-20 20:34:03 +02:00
const mediaProgressRemoved = await Database.mediaProgressModel.destroy({
where: {
id: {
2024-05-05 23:39:38 +02:00
[Sequelize.Op.in]: mediaProgressToRemove.map((mp) => mp.id)
2023-07-05 01:14:44 +02:00
}
}
})
if (mediaProgressRemoved) {
Logger.info(`[Server] Removed ${mediaProgressRemoved} media progress for media items that no longer exist in db`)
}
}
2023-07-05 01:14:44 +02:00
// Remove series from hide from continue listening that no longer exist
2023-08-20 20:34:03 +02:00
const users = await Database.userModel.getOldUsers()
for (const _user of users) {
2023-07-05 01:14:44 +02:00
let hasUpdated = false
if (_user.seriesHideFromContinueListening.length) {
2024-05-05 23:39:38 +02:00
const seriesHiding = (
await Database.seriesModel.findAll({
where: {
id: _user.seriesHideFromContinueListening
},
attributes: ['id'],
raw: true
})
).map((se) => se.id)
_user.seriesHideFromContinueListening = _user.seriesHideFromContinueListening.filter((seriesId) => {
if (!seriesHiding.includes(seriesId)) {
// Series removed
hasUpdated = true
return false
}
return true
})
}
if (hasUpdated) {
2023-07-05 01:14:44 +02:00
await Database.updateUser(_user)
}
}
}
2023-12-28 23:32:21 +01:00
/**
* Gracefully stop server
* Stops watcher and socket server
*/
2021-08-18 00:01:11 +02:00
async stop() {
Logger.info('=== Stopping Server ===')
2021-08-18 00:01:11 +02:00
await this.watcher.close()
Logger.info('Watcher Closed')
return new Promise((resolve) => {
2023-12-27 14:33:33 +01:00
SocketAuthority.close((err) => {
2021-08-18 00:01:11 +02:00
if (err) {
Logger.error('Failed to close server', err)
} else {
Logger.info('Server successfully closed')
}
resolve()
})
})
}
}
module.exports = Server