diff --git a/server/LogManager.js b/server/LogManager.js new file mode 100644 index 00000000..b7590d51 --- /dev/null +++ b/server/LogManager.js @@ -0,0 +1,124 @@ +const Path = require('path') +const fs = require('fs-extra') + +const DailyLog = require('./objects/DailyLog') + +const Logger = require('./Logger') +const { getFileSize } = require('./utils/fileUtils') + +const TAG = '[LogManager]' + +class LogManager { + constructor(MetadataPath, db) { + this.db = db + this.MetadataPath = MetadataPath + + this.logDirPath = Path.join(this.MetadataPath, 'logs') + this.dailyLogDirPath = Path.join(this.logDirPath, 'daily') + + this.currentDailyLog = null + this.dailyLogBuffer = [] + this.dailyLogFiles = [] + } + + get serverSettings() { + return this.db.serverSettings || {} + } + + get loggerDailyLogsToKeep() { + return this.serverSettings.loggerDailyLogsToKeep || 7 + } + + async init() { + // Load daily logs + await this.scanLogFiles() + + // Check remove extra daily logs + if (this.dailyLogFiles.length > this.loggerDailyLogsToKeep) { + var dailyLogFilesCopy = [...this.dailyLogFiles] + for (let i = 0; i < dailyLogFilesCopy.length - this.loggerDailyLogsToKeep; i++) { + var logFileToRemove = dailyLogFilesCopy[i] + await this.removeLogFile(logFileToRemove) + } + } + + var currentDailyLogFilename = DailyLog.getCurrentDailyLogFilename() + Logger.info(TAG, `Init current daily log filename: ${currentDailyLogFilename}`) + + this.currentDailyLog = new DailyLog() + this.currentDailyLog.setData({ dailyLogDirPath: this.dailyLogDirPath }) + + if (this.dailyLogFiles.includes(currentDailyLogFilename)) { + Logger.debug(TAG, `Daily log file already exists - set in Logger`) + this.currentDailyLog.loadLogs() + } else { + this.dailyLogFiles.push(this.currentDailyLog.filename) + } + + // Log buffered Logs + if (this.dailyLogBuffer.length) { + this.dailyLogBuffer.forEach((logObj) => this.currentDailyLog.appendLog(logObj)) + this.dailyLogBuffer = [] + } + } + + async scanLogFiles() { + await fs.ensureDir(this.dailyLogDirPath) + var dailyFiles = await fs.readdir(this.dailyLogDirPath) + if (dailyFiles && dailyFiles.length) { + dailyFiles.forEach((logFile) => { + if (Path.extname(logFile) === '.txt') { + Logger.info('Daily Log file found', logFile) + this.dailyLogFiles.push(logFile) + } else { + Logger.debug(TAG, 'Unknown File in Daily log files dir', logFile) + } + }) + } + this.dailyLogFiles.sort() + } + + async removeOldestLog() { + if (!this.dailyLogFiles.length) return + var oldestLog = this.dailyLogFiles[0] + return this.removeLogFile(oldestLog) + } + + async removeLogFile(filename) { + var fullPath = Path.join(this.dailyLogDirPath, filename) + var exists = await fs.pathExists(fullPath) + if (!exists) { + Logger.error(TAG, 'Invalid log dne ' + fullPath) + this.dailyLogFiles = this.dailyLogFiles.filter(dlf => dlf.filename !== filename) + } else { + try { + await fs.unlink(fullPath) + Logger.info(TAG, 'Removed daily log: ' + filename) + this.dailyLogFiles = this.dailyLogFiles.filter(dlf => dlf.filename !== filename) + } catch (error) { + Logger.error(TAG, 'Failed to unlink log file ' + fullPath) + } + } + } + + logToFile(logObj) { + if (!this.currentDailyLog) { + this.dailyLogBuffer.push(logObj) + return + } + + // Check log rolls to next day + if (this.currentDailyLog.id !== DailyLog.getCurrentDateString()) { + var newDailyLog = new DailyLog() + newDailyLog.setData({ dailyLogDirPath: this.dailyLogDirPath }) + this.currentDailyLog = newDailyLog + if (this.dailyLogFiles.length > this.loggerDailyLogsToKeep) { + this.removeOldestLog() + } + } + + // Append log line to log file + this.currentDailyLog.appendLog(logObj) + } +} +module.exports = LogManager \ No newline at end of file diff --git a/server/Logger.js b/server/Logger.js index 01ae43d3..e27fbfa8 100644 --- a/server/Logger.js +++ b/server/Logger.js @@ -3,7 +3,10 @@ const { LogLevel } = require('./utils/constants') class Logger { constructor() { this.logLevel = process.env.NODE_ENV === 'production' ? LogLevel.INFO : LogLevel.TRACE + this.logFileLevel = LogLevel.INFO this.socketListeners = [] + + this.logManager = null } get timestamp() { @@ -49,15 +52,21 @@ class Logger { this.socketListeners = this.socketListeners.filter(s => s.id !== socketId) } - logToSockets(level, args) { + handleLog(level, args) { + const logObj = { + timestamp: this.timestamp, + message: args.join(' '), + levelName: this.getLogLevelString(level), + level + } + + if (level >= this.logFileLevel && this.logManager) { + this.logManager.logToFile(logObj) + } + this.socketListeners.forEach((socketListener) => { if (socketListener.level <= level) { - socketListener.socket.emit('log', { - timestamp: this.timestamp, - message: args.join(' '), - levelName: this.getLogLevelString(level), - level - }) + socketListener.socket.emit('log', logObj) } }) } @@ -70,41 +79,41 @@ class Logger { trace(...args) { if (this.logLevel > LogLevel.TRACE) return console.trace(`[${this.timestamp}] TRACE:`, ...args) - this.logToSockets(LogLevel.TRACE, args) + this.handleLog(LogLevel.TRACE, args) } debug(...args) { if (this.logLevel > LogLevel.DEBUG) return console.debug(`[${this.timestamp}] DEBUG:`, ...args) - this.logToSockets(LogLevel.DEBUG, args) + this.handleLog(LogLevel.DEBUG, args) } info(...args) { if (this.logLevel > LogLevel.INFO) return console.info(`[${this.timestamp}] INFO:`, ...args) - this.logToSockets(LogLevel.INFO, args) + this.handleLog(LogLevel.INFO, args) } warn(...args) { if (this.logLevel > LogLevel.WARN) return console.warn(`[${this.timestamp}] WARN:`, ...args) - this.logToSockets(LogLevel.WARN, args) + this.handleLog(LogLevel.WARN, args) } error(...args) { if (this.logLevel > LogLevel.ERROR) return console.error(`[${this.timestamp}] ERROR:`, ...args) - this.logToSockets(LogLevel.ERROR, args) + this.handleLog(LogLevel.ERROR, args) } fatal(...args) { console.error(`[${this.timestamp}] FATAL:`, ...args) - this.logToSockets(LogLevel.FATAL, args) + this.handleLog(LogLevel.FATAL, args) } note(...args) { console.log(`[${this.timestamp}] NOTE:`, ...args) - this.logToSockets(LogLevel.NOTE, args) + this.handleLog(LogLevel.NOTE, args) } } module.exports = new Logger() \ No newline at end of file diff --git a/server/Server.js b/server/Server.js index 6ba19f20..6158b424 100644 --- a/server/Server.js +++ b/server/Server.js @@ -20,6 +20,7 @@ const Watcher = require('./Watcher') const Scanner = require('./Scanner') const Db = require('./Db') const BackupManager = require('./BackupManager') +const LogManager = require('./LogManager') const ApiController = require('./ApiController') const HlsController = require('./HlsController') const StreamManager = require('./StreamManager') @@ -44,6 +45,7 @@ class Server { this.db = new Db(this.ConfigPath, this.AudiobookPath) this.auth = new Auth(this.db) this.backupManager = new BackupManager(this.MetadataPath, this.Uid, this.Gid, this.db) + this.logManager = new LogManager(this.MetadataPath, this.db) this.watcher = new Watcher(this.AudiobookPath) this.coverController = new CoverController(this.db, this.MetadataPath, this.AudiobookPath) this.scanner = new Scanner(this.AudiobookPath, this.MetadataPath, this.db, this.coverController, this.emitter.bind(this)) @@ -53,6 +55,8 @@ class Server { this.apiController = new ApiController(this.MetadataPath, this.db, this.scanner, this.auth, this.streamManager, this.rssFeeds, this.downloadManager, this.coverController, this.backupManager, this.watcher, this.emitter.bind(this), this.clientEmitter.bind(this)) this.hlsController = new HlsController(this.db, this.scanner, this.auth, this.streamManager, this.emitter.bind(this), this.streamManager.StreamsPath) + Logger.logManager = this.logManager + this.expressApp = null this.server = null this.io = null @@ -111,6 +115,7 @@ class Server { await this.purgeMetadata() await this.backupManager.init() + await this.logManager.init() this.watcher.initWatcher(this.libraries) this.watcher.on('files', this.filesChanged.bind(this)) diff --git a/server/StreamManager.js b/server/StreamManager.js index 68b6f3a6..8e0a950b 100644 --- a/server/StreamManager.js +++ b/server/StreamManager.js @@ -68,7 +68,7 @@ class StreamManager { if (!dirs || !dirs.length) return true await Promise.all(dirs.map(async (dirname) => { - if (dirname !== 'streams' && dirname !== 'books' && dirname !== 'downloads' && dirname !== 'backups') { + if (dirname !== 'streams' && dirname !== 'books' && dirname !== 'downloads' && dirname !== 'backups' && dirname !== 'logs') { var fullPath = Path.join(this.MetadataPath, dirname) Logger.warn(`Removing OLD Orphan Stream ${dirname}`) return fs.remove(fullPath) diff --git a/server/objects/Backup.js b/server/objects/Backup.js index 116ab1a2..727a9ff6 100644 --- a/server/objects/Backup.js +++ b/server/objects/Backup.js @@ -47,7 +47,6 @@ class Backup { backupMetadataCovers: this.backupMetadataCovers, backupDirPath: this.backupDirPath, datePretty: this.datePretty, - path: this.path, fullPath: this.fullPath, path: this.path, filename: this.filename, diff --git a/server/objects/DailyLog.js b/server/objects/DailyLog.js new file mode 100644 index 00000000..1dc02bed --- /dev/null +++ b/server/objects/DailyLog.js @@ -0,0 +1,112 @@ +const Path = require('path') +const date = require('date-and-time') +const fs = require('fs-extra') +const { readTextFile } = require('../utils/fileUtils') +const Logger = require('../Logger') + +class DailyLog { + constructor() { + this.id = null + this.datePretty = null + + this.dailyLogDirPath = null + this.filename = null + this.path = null + this.fullPath = null + + this.createdAt = null + + this.logs = [] + this.bufferedLogLines = [] + this.locked = false + } + + static getCurrentDailyLogFilename() { + return date.format(new Date(), 'YYYY-MM-DD') + '.txt' + } + + static getCurrentDateString() { + return date.format(new Date(), 'YYYY-MM-DD') + } + + toJSON() { + return { + id: this.id, + datePretty: this.datePretty, + path: this.path, + dailyLogDirPath: this.dailyLogDirPath, + fullPath: this.fullPath, + filename: this.filename, + createdAt: this.createdAt + } + } + + setData(data) { + this.id = date.format(new Date(), 'YYYY-MM-DD') + this.datePretty = date.format(new Date(), 'ddd, MMM D YYYY') + + this.dailyLogDirPath = data.dailyLogDirPath + + this.filename = this.id + '.txt' + this.path = Path.join('backups', this.filename) + this.fullPath = Path.join(this.dailyLogDirPath, this.filename) + + this.createdAt = Date.now() + } + + async appendBufferedLogs() { + var buffered = [...this.bufferedLogLines] + this.bufferedLogLines = [] + + var oneBigLog = '' + buffered.forEach((logLine) => { + oneBigLog += logLine + '\n' + }) + + this.appendLogLine(oneBigLog) + } + + async appendLog(logObj) { + this.logs.push(logObj) + var line = JSON.stringify(logObj) + this.appendLogLine(line) + } + + async appendLogLine(line) { + if (this.locked) { + this.bufferedLogLines.push(line) + return + } + this.locked = true + + await fs.writeFile(this.fullPath, line, { flag: "a+" }).catch((error) => { + console.log('[DailyLog] Append log failed', error) + }) + + this.locked = false + if (this.bufferedLogLines.length) { + this.appendBufferedLogs() + } + } + + async loadLogs() { + var exists = await fs.pathExists(this.fullPath) + if (!exists) { + console.error('Daily log does not exist') + return + } + + var text = await readTextFile(this.fullPath) + this.logs = text.split(/\r?\n/).map(t => { + try { + return JSON.parse(t) + } catch (err) { + console.error('Failed to parse log line', t, err) + return null + } + }).filter(l => !!l) + + Logger.info(`[DailyLog] ${this.id}: Loaded ${this.logs.length} Logs`) + } +} +module.exports = DailyLog \ No newline at end of file diff --git a/server/objects/ServerSettings.js b/server/objects/ServerSettings.js index 517db213..6b368dcd 100644 --- a/server/objects/ServerSettings.js +++ b/server/objects/ServerSettings.js @@ -27,6 +27,10 @@ class ServerSettings { this.backupsToKeep = 2 this.backupMetadataCovers = true + // Logger + this.loggerDailyLogsToKeep = 7 + this.loggerScannerLogsToKeep = 2 + this.logLevel = Logger.logLevel if (settings) { @@ -48,6 +52,9 @@ class ServerSettings { this.backupsToKeep = settings.backupsToKeep || 2 this.backupMetadataCovers = settings.backupMetadataCovers !== false + this.loggerDailyLogsToKeep = settings.loggerDailyLogsToKeep || 7 + this.loggerScannerLogsToKeep = settings.loggerScannerLogsToKeep || 2 + this.logLevel = settings.logLevel || Logger.logLevel if (this.logLevel !== Logger.logLevel) { @@ -69,6 +76,8 @@ class ServerSettings { backupSchedule: this.backupSchedule, backupsToKeep: this.backupsToKeep, backupMetadataCovers: this.backupMetadataCovers, + loggerDailyLogsToKeep: this.loggerDailyLogsToKeep, + loggerScannerLogsToKeep: this.loggerScannerLogsToKeep, logLevel: this.logLevel } }