Add: Daily file logs and log server settings

server/LogManager.js

@@ -0,0 +1,124 @@
+const Path = require('path')
+const fs = require('fs-extra')
+
+const DailyLog = require('./objects/DailyLog')
+
+const Logger = require('./Logger')
+const { getFileSize } = require('./utils/fileUtils')
+
+const TAG = '[LogManager]'
+
+class LogManager {
+  constructor(MetadataPath, db) {
+    this.db = db
+    this.MetadataPath = MetadataPath
+
+    this.logDirPath = Path.join(this.MetadataPath, 'logs')
+    this.dailyLogDirPath = Path.join(this.logDirPath, 'daily')
+
+    this.currentDailyLog = null
+    this.dailyLogBuffer = []
+    this.dailyLogFiles = []
+  }
+
+  get serverSettings() {
+    return this.db.serverSettings || {}
+  }
+
+  get loggerDailyLogsToKeep() {
+    return this.serverSettings.loggerDailyLogsToKeep || 7
+  }
+
+  async init() {
+    // Load daily logs
+    await this.scanLogFiles()
+
+    // Check remove extra daily logs
+    if (this.dailyLogFiles.length > this.loggerDailyLogsToKeep) {
+      var dailyLogFilesCopy = [...this.dailyLogFiles]
+      for (let i = 0; i < dailyLogFilesCopy.length - this.loggerDailyLogsToKeep; i++) {
+        var logFileToRemove = dailyLogFilesCopy[i]
+        await this.removeLogFile(logFileToRemove)
+      }
+    }
+
+    var currentDailyLogFilename = DailyLog.getCurrentDailyLogFilename()
+    Logger.info(TAG, `Init current daily log filename: ${currentDailyLogFilename}`)
+
+    this.currentDailyLog = new DailyLog()
+    this.currentDailyLog.setData({ dailyLogDirPath: this.dailyLogDirPath })
+
+    if (this.dailyLogFiles.includes(currentDailyLogFilename)) {
+      Logger.debug(TAG, `Daily log file already exists - set in Logger`)
+      this.currentDailyLog.loadLogs()
+    } else {
+      this.dailyLogFiles.push(this.currentDailyLog.filename)
+    }
+
+    // Log buffered Logs
+    if (this.dailyLogBuffer.length) {
+      this.dailyLogBuffer.forEach((logObj) => this.currentDailyLog.appendLog(logObj))
+      this.dailyLogBuffer = []
+    }
+  }
+
+  async scanLogFiles() {
+    await fs.ensureDir(this.dailyLogDirPath)
+    var dailyFiles = await fs.readdir(this.dailyLogDirPath)
+    if (dailyFiles && dailyFiles.length) {
+      dailyFiles.forEach((logFile) => {
+        if (Path.extname(logFile) === '.txt') {
+          Logger.info('Daily Log file found', logFile)
+          this.dailyLogFiles.push(logFile)
+        } else {
+          Logger.debug(TAG, 'Unknown File in Daily log files dir', logFile)
+        }
+      })
+    }
+    this.dailyLogFiles.sort()
+  }
+
+  async removeOldestLog() {
+    if (!this.dailyLogFiles.length) return
+    var oldestLog = this.dailyLogFiles[0]
+    return this.removeLogFile(oldestLog)
+  }
+
+  async removeLogFile(filename) {
+    var fullPath = Path.join(this.dailyLogDirPath, filename)
+    var exists = await fs.pathExists(fullPath)
+    if (!exists) {
+      Logger.error(TAG, 'Invalid log dne ' + fullPath)
+      this.dailyLogFiles = this.dailyLogFiles.filter(dlf => dlf.filename !== filename)
+    } else {
+      try {
+        await fs.unlink(fullPath)
+        Logger.info(TAG, 'Removed daily log: ' + filename)
+        this.dailyLogFiles = this.dailyLogFiles.filter(dlf => dlf.filename !== filename)
+      } catch (error) {
+        Logger.error(TAG, 'Failed to unlink log file ' + fullPath)
+      }
+    }
+  }
+
+  logToFile(logObj) {
+    if (!this.currentDailyLog) {
+      this.dailyLogBuffer.push(logObj)
+      return
+    }
+
+    // Check log rolls to next day
+    if (this.currentDailyLog.id !== DailyLog.getCurrentDateString()) {
+      var newDailyLog = new DailyLog()
+      newDailyLog.setData({ dailyLogDirPath: this.dailyLogDirPath })
+      this.currentDailyLog = newDailyLog
+      if (this.dailyLogFiles.length > this.loggerDailyLogsToKeep) {
+        this.removeOldestLog()
+      }
+    }
+
+    // Append log line to log file
+    this.currentDailyLog.appendLog(logObj)
+  }
+}
+module.exports = LogManager

server/Logger.js

@@ -3,7 +3,10 @@ const { LogLevel } = require('./utils/constants')
 class Logger {
   constructor() {
     this.logLevel = process.env.NODE_ENV === 'production' ? LogLevel.INFO : LogLevel.TRACE
+    this.logFileLevel = LogLevel.INFO
     this.socketListeners = []
+
+    this.logManager = null
   }
 
   get timestamp() {
@@ -49,15 +52,21 @@ class Logger {
     this.socketListeners = this.socketListeners.filter(s => s.id !== socketId)
   }
 
-  logToSockets(level, args) {
+  handleLog(level, args) {
-    this.socketListeners.forEach((socketListener) => {
+    const logObj = {
-      if (socketListener.level <= level) {
-        socketListener.socket.emit('log', {
       timestamp: this.timestamp,
       message: args.join(' '),
       levelName: this.getLogLevelString(level),
       level
-        })
+    }
+
+    if (level >= this.logFileLevel && this.logManager) {
+      this.logManager.logToFile(logObj)
+    }
+
+    this.socketListeners.forEach((socketListener) => {
+      if (socketListener.level <= level) {
+        socketListener.socket.emit('log', logObj)
       }
     })
   }
@@ -70,41 +79,41 @@ class Logger {
   trace(...args) {
     if (this.logLevel > LogLevel.TRACE) return
     console.trace(`[${this.timestamp}] TRACE:`, ...args)
-    this.logToSockets(LogLevel.TRACE, args)
+    this.handleLog(LogLevel.TRACE, args)
   }
 
   debug(...args) {
     if (this.logLevel > LogLevel.DEBUG) return
     console.debug(`[${this.timestamp}] DEBUG:`, ...args)
-    this.logToSockets(LogLevel.DEBUG, args)
+    this.handleLog(LogLevel.DEBUG, args)
   }
 
   info(...args) {
     if (this.logLevel > LogLevel.INFO) return
     console.info(`[${this.timestamp}]  INFO:`, ...args)
-    this.logToSockets(LogLevel.INFO, args)
+    this.handleLog(LogLevel.INFO, args)
   }
 
   warn(...args) {
     if (this.logLevel > LogLevel.WARN) return
     console.warn(`[${this.timestamp}]  WARN:`, ...args)
-    this.logToSockets(LogLevel.WARN, args)
+    this.handleLog(LogLevel.WARN, args)
   }
 
   error(...args) {
     if (this.logLevel > LogLevel.ERROR) return
     console.error(`[${this.timestamp}] ERROR:`, ...args)
-    this.logToSockets(LogLevel.ERROR, args)
+    this.handleLog(LogLevel.ERROR, args)
   }
 
   fatal(...args) {
     console.error(`[${this.timestamp}] FATAL:`, ...args)
-    this.logToSockets(LogLevel.FATAL, args)
+    this.handleLog(LogLevel.FATAL, args)
   }
 
   note(...args) {
     console.log(`[${this.timestamp}] NOTE:`, ...args)
-    this.logToSockets(LogLevel.NOTE, args)
+    this.handleLog(LogLevel.NOTE, args)
   }
 }
 module.exports = new Logger()

server/Server.js

@@ -20,6 +20,7 @@ const Watcher = require('./Watcher')
 const Scanner = require('./Scanner')
 const Db = require('./Db')
 const BackupManager = require('./BackupManager')
+const LogManager = require('./LogManager')
 const ApiController = require('./ApiController')
 const HlsController = require('./HlsController')
 const StreamManager = require('./StreamManager')
@@ -44,6 +45,7 @@ class Server {
     this.db = new Db(this.ConfigPath, this.AudiobookPath)
     this.auth = new Auth(this.db)
     this.backupManager = new BackupManager(this.MetadataPath, this.Uid, this.Gid, this.db)
+    this.logManager = new LogManager(this.MetadataPath, this.db)
     this.watcher = new Watcher(this.AudiobookPath)
     this.coverController = new CoverController(this.db, this.MetadataPath, this.AudiobookPath)
     this.scanner = new Scanner(this.AudiobookPath, this.MetadataPath, this.db, this.coverController, this.emitter.bind(this))
@@ -53,6 +55,8 @@ class Server {
     this.apiController = new ApiController(this.MetadataPath, this.db, this.scanner, this.auth, this.streamManager, this.rssFeeds, this.downloadManager, this.coverController, this.backupManager, this.watcher, this.emitter.bind(this), this.clientEmitter.bind(this))
     this.hlsController = new HlsController(this.db, this.scanner, this.auth, this.streamManager, this.emitter.bind(this), this.streamManager.StreamsPath)
 
+    Logger.logManager = this.logManager
+
     this.expressApp = null
     this.server = null
     this.io = null
@@ -111,6 +115,7 @@ class Server {
 
     await this.purgeMetadata()
     await this.backupManager.init()
+    await this.logManager.init()
 
     this.watcher.initWatcher(this.libraries)
     this.watcher.on('files', this.filesChanged.bind(this))

server/StreamManager.js

@@ -68,7 +68,7 @@ class StreamManager {
       if (!dirs || !dirs.length) return true
 
       await Promise.all(dirs.map(async (dirname) => {
-        if (dirname !== 'streams' && dirname !== 'books' && dirname !== 'downloads' && dirname !== 'backups') {
+        if (dirname !== 'streams' && dirname !== 'books' && dirname !== 'downloads' && dirname !== 'backups' && dirname !== 'logs') {
           var fullPath = Path.join(this.MetadataPath, dirname)
           Logger.warn(`Removing OLD Orphan Stream ${dirname}`)
           return fs.remove(fullPath)

server/objects/Backup.js

@@ -47,7 +47,6 @@ class Backup {
       backupMetadataCovers: this.backupMetadataCovers,
       backupDirPath: this.backupDirPath,
       datePretty: this.datePretty,
-      path: this.path,
       fullPath: this.fullPath,
       path: this.path,
       filename: this.filename,

server/objects/DailyLog.js

@@ -0,0 +1,112 @@
+const Path = require('path')
+const date = require('date-and-time')
+const fs = require('fs-extra')
+const { readTextFile } = require('../utils/fileUtils')
+const Logger = require('../Logger')
+
+class DailyLog {
+  constructor() {
+    this.id = null
+    this.datePretty = null
+
+    this.dailyLogDirPath = null
+    this.filename = null
+    this.path = null
+    this.fullPath = null
+
+    this.createdAt = null
+
+    this.logs = []
+    this.bufferedLogLines = []
+    this.locked = false
+  }
+
+  static getCurrentDailyLogFilename() {
+    return date.format(new Date(), 'YYYY-MM-DD') + '.txt'
+  }
+
+  static getCurrentDateString() {
+    return date.format(new Date(), 'YYYY-MM-DD')
+  }
+
+  toJSON() {
+    return {
+      id: this.id,
+      datePretty: this.datePretty,
+      path: this.path,
+      dailyLogDirPath: this.dailyLogDirPath,
+      fullPath: this.fullPath,
+      filename: this.filename,
+      createdAt: this.createdAt
+    }
+  }
+
+  setData(data) {
+    this.id = date.format(new Date(), 'YYYY-MM-DD')
+    this.datePretty = date.format(new Date(), 'ddd, MMM D YYYY')
+
+    this.dailyLogDirPath = data.dailyLogDirPath
+
+    this.filename = this.id + '.txt'
+    this.path = Path.join('backups', this.filename)
+    this.fullPath = Path.join(this.dailyLogDirPath, this.filename)
+
+    this.createdAt = Date.now()
+  }
+
+  async appendBufferedLogs() {
+    var buffered = [...this.bufferedLogLines]
+    this.bufferedLogLines = []
+
+    var oneBigLog = ''
+    buffered.forEach((logLine) => {
+      oneBigLog += logLine + '\n'
+    })
+
+    this.appendLogLine(oneBigLog)
+  }
+
+  async appendLog(logObj) {
+    this.logs.push(logObj)
+    var line = JSON.stringify(logObj)
+    this.appendLogLine(line)
+  }
+
+  async appendLogLine(line) {
+    if (this.locked) {
+      this.bufferedLogLines.push(line)
+      return
+    }
+    this.locked = true
+
+    await fs.writeFile(this.fullPath, line, { flag: "a+" }).catch((error) => {
+      console.log('[DailyLog] Append log failed', error)
+    })
+
+    this.locked = false
+    if (this.bufferedLogLines.length) {
+      this.appendBufferedLogs()
+    }
+  }
+
+  async loadLogs() {
+    var exists = await fs.pathExists(this.fullPath)
+    if (!exists) {
+      console.error('Daily log does not exist')
+      return
+    }
+
+    var text = await readTextFile(this.fullPath)
+    this.logs = text.split(/\r?\n/).map(t => {
+      try {
+        return JSON.parse(t)
+      } catch (err) {
+        console.error('Failed to parse log line', t, err)
+        return null
+      }
+    }).filter(l => !!l)
+
+    Logger.info(`[DailyLog] ${this.id}: Loaded ${this.logs.length} Logs`)
+  }
+}
+module.exports = DailyLog

server/objects/ServerSettings.js

@@ -27,6 +27,10 @@ class ServerSettings {
     this.backupsToKeep = 2
     this.backupMetadataCovers = true
 
+    // Logger
+    this.loggerDailyLogsToKeep = 7
+    this.loggerScannerLogsToKeep = 2
+
     this.logLevel = Logger.logLevel
 
     if (settings) {
@@ -48,6 +52,9 @@ class ServerSettings {
     this.backupsToKeep = settings.backupsToKeep || 2
     this.backupMetadataCovers = settings.backupMetadataCovers !== false
 
+    this.loggerDailyLogsToKeep = settings.loggerDailyLogsToKeep || 7
+    this.loggerScannerLogsToKeep = settings.loggerScannerLogsToKeep || 2
+
     this.logLevel = settings.logLevel || Logger.logLevel
 
     if (this.logLevel !== Logger.logLevel) {
@@ -69,6 +76,8 @@ class ServerSettings {
       backupSchedule: this.backupSchedule,
       backupsToKeep: this.backupsToKeep,
       backupMetadataCovers: this.backupMetadataCovers,
+      loggerDailyLogsToKeep: this.loggerDailyLogsToKeep,
+      loggerScannerLogsToKeep: this.loggerScannerLogsToKeep,
       logLevel: this.logLevel
     }
   }