Mirrors/audiobookshelf
Mirrors/audiobookshelf
1
0
Fork 0
mirror of https://github.com/advplyr/audiobookshelf.git synced 2024-12-20 19:06:06 +01:00
Code Issues Packages Projects Releases Wiki Activity

Update scanner v3, add isActive support for users

Browse Source
This commit is contained in:
advplyr 2021-09-10 19:55:02 -05:00
parent 394d312282
commit beaa1e14bb
13 changed files with 230 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
client/components/app/BookShelf.vue
View File

@ -13,7 +13,7 @@
<p class="text-center text-2xl font-book mb-4">Your Audiobookshelf is empty!</p>
<ui-btn color="success" @click="scan">Scan your Audiobooks</ui-btn>
</div>
<div class="w-full flex flex-col items-center">
<div v-else class="w-full flex flex-col items-center">
<template v-for="(shelf, index) in groupedBooks">
<div :key="index" class="w-full bookshelfRow relative">
<div class="flex justify-center items-center">

2
client/package.json
View File

@ -1,6 +1,6 @@
{
"name": "audiobookshelf-client",
"version": "1.1.2",
"version": "1.1.3",
"description": "Audiobook manager and player",
"main": "index.js",
"scripts": {

2
client/pages/config/index.vue
View File

@ -17,7 +17,7 @@
<th style="width: 200px">Created At</th>
<th style="width: 100px"></th>
</tr>
<tr v-for="user in users" :key="user.id">
<tr v-for="user in users" :key="user.id" :class="user.isActive ? '' : 'bg-error bg-opacity-20'">
<td>
{{ user.username }} <span class="text-xs text-gray-400 italic pl-4">({{ user.id }})</span>
</td>

5
client/store/audiobooks.js
View File

@ -1,8 +1,6 @@
import { sort } from '@/assets/fastSort'
import { decode } from '@/plugins/init.client'
// const STANDARD_GENRES = ['adventure', 'autobiography', 'biography', 'childrens', 'comedy', 'crime', 'dystopian', 'fantasy', 'fiction', 'health', 'history', 'horror', 'mystery', 'new_adult', 'nonfiction', 'philosophy', 'politics', 'religion', 'romance', 'sci-fi', 'self-help', 'short_story', 'technology', 'thriller', 'true_crime', 'western', 'young_adult']
const STANDARD_GENRES = ['Adventure', 'Autobiography', 'Biography', 'Childrens', 'Comedy', 'Crime', 'Dystopian', 'Fantasy', 'Fiction', 'Health', 'History', 'Horror', 'Mystery', 'New Adult', 'Nonfiction', 'Philosophy', 'Politics', 'Religion', 'Romance', 'Sci-Fi', 'Self-Help', 'Short Story', 'Technology', 'Thriller', 'True Crime', 'Western', 'Young Adult']
export const state = () => ({
@ -31,9 +29,10 @@ export const getters = {
}
if (state.keywordFilter) {
const keywordFilterKeys = ['title', 'subtitle', 'author', 'series', 'narrarator']
const keyworkFilter = state.keywordFilter.toLowerCase()
return filtered.filter(ab => {
if (!ab.book) return false
return !!keywordFilterKeys.find(key => (ab.book[key] && ab.book[key].includes(state.keywordFilter)))
return !!keywordFilterKeys.find(key => (ab.book[key] && ab.book[key].toLowerCase().includes(keyworkFilter)))
})
}
return filtered

2
package.json
View File

@ -1,6 +1,6 @@
{
"name": "audiobookshelf",
"version": "1.1.2",
"version": "1.1.3",
"description": "Self-hosted audiobook server for managing and playing audiobooks.",
"main": "index.js",
"scripts": {

8
server/Auth.js
View File

@ -48,6 +48,10 @@ class Auth {
var user = await this.verifyToken(token)
if (!user) {
Logger.error('Verify Token User Not Found', token)
return res.sendStatus(404)
}
if (!user.isActive) {
Logger.error('Verify Token User is disabled', token, user.username)
return res.sendStatus(403)
}
req.user = user
@ -95,6 +99,10 @@ class Auth {
return res.json({ error: 'User not found' })
}
if (!user.isActive) {
return res.json({ error: 'User unavailable' })
}
// Check passwordless root user
if (user.id === 'root' && (!user.pash || user.pash === '')) {
if (password) {

80
server/Scanner.js
View File

@ -1,9 +1,10 @@
const fs = require('fs-extra')
const Path = require('path')
const Logger = require('./Logger')
const BookFinder = require('./BookFinder')
const Audiobook = require('./objects/Audiobook')
const audioFileScanner = require('./utils/audioFileScanner')
const { getAllAudiobookFileData, getAudiobookFileData } = require('./utils/scandir')
const { groupFilesIntoAudiobookPaths, getAudiobookFileData, scanRootDir } = require('./utils/scandir')
const { comparePaths, getIno } = require('./utils/index')
const { secondsToTimestamp } = require('./utils/fileUtils')
const { ScanResult } = require('./utils/constants')
@ -191,7 +192,7 @@ class Scanner {
}
const scanStart = Date.now()
var audiobookDataFound = await getAllAudiobookFileData(this.AudiobookPath, this.db.serverSettings)
var audiobookDataFound = await scanRootDir(this.AudiobookPath, this.db.serverSettings)
// Set ino for each ab data as a string
audiobookDataFound = await this.setAudiobookDataInos(audiobookDataFound)
@ -251,20 +252,7 @@ class Scanner {
}
async scanAudiobook(audiobookPath) {
var exists = await fs.pathExists(audiobookPath)
if (!exists) {
// Audiobook was deleted, TODO: Should confirm this better
var audiobook = this.db.audiobooks.find(ab => ab.fullPath === audiobookPath)
if (audiobook) {
var audiobookJSON = audiobook.toJSONMinified()
await this.db.removeEntity('audiobook', audiobook.id)
this.emitter('audiobook_removed', audiobookJSON)
return ScanResult.REMOVED
}
Logger.warn('Path was deleted but no audiobook found', audiobookPath)
return ScanResult.NOTHING
}
Logger.debug('[Scanner] scanAudiobook', audiobookPath)
var audiobookData = await getAudiobookFileData(this.AudiobookPath, audiobookPath, this.db.serverSettings)
if (!audiobookData) {
return ScanResult.NOTHING
@ -273,6 +261,66 @@ class Scanner {
return this.scanAudiobookData(audiobookData)
}
// Files were modified in this directory, check it out
async checkDir(dir) {
var exists = await fs.pathExists(dir)
if (!exists) {
// Audiobook was deleted, TODO: Should confirm this better
var audiobook = this.db.audiobooks.find(ab => ab.fullPath === dir)
if (audiobook) {
var audiobookJSON = audiobook.toJSONMinified()
await this.db.removeEntity('audiobook', audiobook.id)
this.emitter('audiobook_removed', audiobookJSON)
return ScanResult.REMOVED
}
// Path inside audiobook was deleted, scan audiobook
audiobook = this.db.audiobooks.find(ab => dir.startsWith(ab.fullPath))
if (audiobook) {
Logger.info(`[Scanner] Path inside audiobook "${audiobook.title}" was deleted: ${dir}`)
return this.scanAudiobook(audiobook.fullPath)
}
Logger.warn('[Scanner] Path was deleted but no audiobook found', dir)
return ScanResult.NOTHING
}
// Check if this is a subdirectory of an audiobook
var audiobook = this.db.audiobooks.find((ab) => dir.startsWith(ab.fullPath))
if (audiobook) {
Logger.debug(`[Scanner] Check Dir audiobook "${audiobook.title}" found: ${dir}`)
return this.scanAudiobook(audiobook.fullPath)
}
// Check if an audiobook is a subdirectory of this dir
audiobook = this.db.audiobooks.find(ab => ab.fullPath.startsWith(dir))
if (audiobook) {
Logger.warn(`[Scanner] Files were added/updated in a root directory of an existing audiobook, ignore files: ${dir}`)
return ScanResult.NOTHING
}
// Must be a new audiobook
Logger.debug(`[Scanner] Check Dir must be a new audiobook: ${dir}`)
return this.scanAudiobook(dir)
}
// Array of files that may have been renamed, removed or added
async filesChanged(filepaths) {
if (!filepaths.length) return ScanResult.NOTHING
var relfilepaths = filepaths.map(path => path.replace(this.AudiobookPath, ''))
var fileGroupings = groupFilesIntoAudiobookPaths(relfilepaths)
var results = []
for (const dir in fileGroupings) {
Logger.debug(`[Scanner] Check dir ${dir}`)
var fullPath = Path.join(this.AudiobookPath, dir)
var result = await this.checkDir(fullPath)
Logger.debug(`[Scanner] Check dir result ${result}`)
results.push(result)
}
return results
}
async fetchMetadata(id, trackIndex = 0) {
var audiobook = this.audiobooks.find(a => a.id === id)
if (!audiobook) {

22
server/Server.js
View File

@ -75,20 +75,10 @@ class Server {
})
}
async newFilesAdded({ dir, files }) {
Logger.info(files.length, 'New Files Added in dir', dir)
var result = await this.scanner.scanAudiobook(dir)
Logger.info('New Files Added result', result)
}
async filesRemoved({ dir, files }) {
Logger.info(files.length, 'Files Removed in dir', dir)
var result = await this.scanner.scanAudiobook(dir)
Logger.info('Files Removed result', result)
}
async filesRenamed({ dir, files }) {
Logger.info(files.length, 'Files Renamed in dir', dir)
var result = await this.scanner.scanAudiobook(dir)
Logger.info('Files Renamed result', result)
async filesChanged(files) {
Logger.info('[Server]', files.length, 'Files Changed')
var result = await this.scanner.filesChanged(files)
Logger.info('[Server] Files changed result', result)
}
async scan() {
@ -125,9 +115,7 @@ class Server {
this.auth.init()
this.watcher.initWatcher()
this.watcher.on('new_files', this.newFilesAdded.bind(this))
this.watcher.on('removed_files', this.filesRemoved.bind(this))
this.watcher.on('renamed_files', this.filesRenamed.bind(this))
this.watcher.on('files', this.filesChanged.bind(this))
}
authMiddleware(req, res, next) {

74
server/Watcher.js
View File

@ -2,7 +2,6 @@ const Path = require('path')
const EventEmitter = require('events')
const Watcher = require('watcher')
const Logger = require('./Logger')
const { getIno } = require('./utils/index')
class FolderWatcher extends EventEmitter {
constructor(audiobookPath) {
@ -11,10 +10,9 @@ class FolderWatcher extends EventEmitter {
this.folderMap = {}
this.watcher = null
this.pendingBatchDelay = 4000
// Audiobook paths with changes
this.pendingBatch = {}
this.pendingFiles = []
this.pendingDelay = 4000
this.pendingTimeout = null
}
initWatcher() {
@ -46,7 +44,6 @@ class FolderWatcher extends EventEmitter {
} catch (error) {
Logger.error('Chokidar watcher failed', error)
}
}
close() {
@ -55,43 +52,39 @@ class FolderWatcher extends EventEmitter {
// After [pendingBatchDelay] seconds emit batch
async onNewFile(path) {
if (this.pendingFiles.includes(path)) return
Logger.debug('FolderWatcher: New File', path)
var dir = Path.dirname(path)
if (this.pendingBatch[dir]) {
this.pendingBatch[dir].files.push(path)
clearTimeout(this.pendingBatch[dir].timeout)
} else {
this.pendingBatch[dir] = {
dir,
files: [path]
}
if (dir === this.AudiobookPath) {
Logger.debug('New File added to root dir, ignoring it')
return
}
this.pendingBatch[dir].timeout = setTimeout(() => {
this.emit('new_files', this.pendingBatch[dir])
delete this.pendingBatch[dir]
}, this.pendingBatchDelay)
this.pendingFiles.push(path)
clearTimeout(this.pendingTimeout)
this.pendingTimeout = setTimeout(() => {
this.emit('files', this.pendingFiles.map(f => f))
this.pendingFiles = []
}, this.pendingDelay)
}
onFileRemoved(path) {
Logger.debug('[FolderWatcher] File Removed', path)
var dir = Path.dirname(path)
if (this.pendingBatch[dir]) {
this.pendingBatch[dir].files.push(path)
clearTimeout(this.pendingBatch[dir].timeout)
} else {
this.pendingBatch[dir] = {
dir,
files: [path]
}
if (dir === this.AudiobookPath) {
Logger.debug('New File added to root dir, ignoring it')
return
}
this.pendingBatch[dir].timeout = setTimeout(() => {
this.emit('removed_files', this.pendingBatch[dir])
delete this.pendingBatch[dir]
}, this.pendingBatchDelay)
this.pendingFiles.push(path)
clearTimeout(this.pendingTimeout)
this.pendingTimeout = setTimeout(() => {
this.emit('files', this.pendingFiles.map(f => f))
this.pendingFiles = []
}, this.pendingDelay)
}
onFileUpdated(path) {
@ -102,20 +95,17 @@ class FolderWatcher extends EventEmitter {
Logger.debug(`[FolderWatcher] Rename ${pathFrom} => ${pathTo}`)
var dir = Path.dirname(pathTo)
if (this.pendingBatch[dir]) {
this.pendingBatch[dir].files.push(pathTo)
clearTimeout(this.pendingBatch[dir].timeout)
} else {
this.pendingBatch[dir] = {
dir,
files: [pathTo]
}
if (dir === this.AudiobookPath) {
Logger.debug('New File added to root dir, ignoring it')
return
}
this.pendingBatch[dir].timeout = setTimeout(() => {
this.emit('renamed_files', this.pendingBatch[dir])
delete this.pendingBatch[dir]
}, this.pendingBatchDelay)
this.pendingFiles.push(pathTo)
clearTimeout(this.pendingTimeout)
this.pendingTimeout = setTimeout(() => {
this.emit('files', this.pendingFiles.map(f => f))
this.pendingFiles = []
}, this.pendingDelay)
}
}
module.exports = FolderWatcher

6
server/objects/User.js
View File

@ -24,13 +24,13 @@ class User {
return this.type === 'root'
}
get canDelete() {
return !!this.permissions.delete
return !!this.permissions.delete && this.isActive
}
get canUpdate() {
return !!this.permissions.update
return !!this.permissions.update && this.isActive
}
get canDownload() {
return !!this.permissions.download
return !!this.permissions.download && this.isActive
}
getDefaultUserSettings() {

1
server/utils/audioFileScanner.js
View File

@ -91,7 +91,6 @@ async function scanAudioFiles(audiobook, newAudioFiles) {
var tracks = []
for (let i = 0; i < newAudioFiles.length; i++) {
var audioFile = newAudioFiles[i]
var scanData = await scan(audioFile.fullPath)
if (!scanData || scanData.error) {
Logger.error('[AudioFileScanner] Scan failed for', audioFile.path)

2
server/utils/index.js
View File

@ -59,7 +59,7 @@ module.exports.comparePaths = (path1, path2) => {
module.exports.getIno = (path) => {
return fs.promises.stat(path, { bigint: true }).then((data => String(data.ino))).catch((err) => {
Logger.error('[Utils] Failed to get ino for path', path, error)
Logger.error('[Utils] Failed to get ino for path', path, err)
return null
})
}

184
server/utils/scandir.js
View File

@ -1,7 +1,6 @@
const Path = require('path')
const dir = require('node-dir')
const Logger = require('../Logger')
const { cleanString } = require('./index')
const AUDIO_FORMATS = ['m4b', 'mp3', 'm4a']
const INFO_FORMATS = ['nfo']
@ -12,7 +11,7 @@ function getPaths(path) {
return new Promise((resolve) => {
dir.paths(path, function (err, res) {
if (err) {
console.error(err)
Logger.error(err)
resolve(false)
}
resolve(res)
@ -20,6 +19,54 @@ function getPaths(path) {
})
}
function groupFilesIntoAudiobookPaths(paths) {
// Step 1: Normalize path, Remove leading "/", Filter out files in root dir
var pathsFiltered = paths.map(path => Path.normalize(path.slice(1))).filter(path => Path.parse(path).dir)
// Step 2: Sort by least number of directories
pathsFiltered.sort((a, b) => {
var pathsA = Path.dirname(a).split(Path.sep).length
var pathsB = Path.dirname(b).split(Path.sep).length
return pathsA - pathsB
})
// Step 3: Group into audiobooks
var audiobookGroup = {}
pathsFiltered.forEach((path) => {
var dirparts = Path.dirname(path).split(Path.sep)
var numparts = dirparts.length
var _path = ''
for (let i = 0; i < numparts; i++) {
var dirpart = dirparts.shift()
_path = Path.join(_path, dirpart)
if (audiobookGroup[_path]) {
var relpath = Path.join(dirparts.join(Path.sep), Path.basename(path))
audiobookGroup[_path].push(relpath)
return
} else if (!dirparts.length) {
audiobookGroup[_path] = [Path.basename(path)]
return
}
}
})
return audiobookGroup
}
module.exports.groupFilesIntoAudiobookPaths = groupFilesIntoAudiobookPaths
function cleanFileObjects(basepath, abrelpath, files) {
return files.map((file) => {
var ext = Path.extname(file)
return {
filetype: getFileType(ext),
filename: Path.basename(file),
path: Path.join(abrelpath, file), // /AUDIOBOOK/PATH/filename.mp3
fullPath: Path.join(basepath, file), // /audiobooks/AUDIOBOOK/PATH/filename.mp3
ext: ext
}
})
}
function getFileType(ext) {
var ext_cleaned = ext.toLowerCase()
if (ext_cleaned.startsWith('.')) ext_cleaned = ext_cleaned.slice(1)
@ -30,27 +77,53 @@ function getFileType(ext) {
return 'unknown'
}
// Input relative filepath, output all details that can be parsed
function getAudiobookDataFromFilepath(abRootPath, relpath, parseSubtitle = false) {
var pathformat = Path.parse(relpath)
var path = pathformat.dir
// Primary scan: abRootPath is /audiobooks
async function scanRootDir(abRootPath, serverSettings = {}) {
var parseSubtitle = !!serverSettings.scannerParseSubtitle
if (!path) {
Logger.error('Ignoring file in root dir', relpath)
return null
var pathdata = await getPaths(abRootPath)
var filepaths = pathdata.files.map(filepath => {
return Path.normalize(filepath).replace(abRootPath, '')
})
var audiobookGrouping = groupFilesIntoAudiobookPaths(filepaths)
if (!Object.keys(audiobookGrouping).length) {
Logger.error('Root path has no audiobooks')
return []
}
// If relative file directory has 3 folders, then the middle folder will be series
var splitDir = path.split(Path.sep)
var author = null
if (splitDir.length > 1) author = splitDir.shift()
var audiobooks = []
for (const audiobookPath in audiobookGrouping) {
var audiobookData = getAudiobookDataFromDir(abRootPath, audiobookPath, parseSubtitle)
var fileObjs = cleanFileObjects(audiobookData.fullPath, audiobookPath, audiobookGrouping[audiobookPath])
audiobooks.push({
...audiobookData,
audioFiles: fileObjs.filter(f => f.filetype === 'audio'),
otherFiles: fileObjs.filter(f => f.filetype !== 'audio')
})
}
return audiobooks
}
module.exports.scanRootDir = scanRootDir
// Input relative filepath, output all details that can be parsed
function getAudiobookDataFromDir(abRootPath, dir, parseSubtitle = false) {
var splitDir = dir.split(Path.sep)
// Audio files will always be in the directory named for the title
var title = splitDir.pop()
var series = null
if (splitDir.length > 1) series = splitDir.shift()
var title = splitDir.shift()
var author = null
// If there are at least 2 more directories, next furthest will be the series
if (splitDir.length > 1) series = splitDir.pop()
if (splitDir.length > 0) author = splitDir.pop()
// There could be many more directories, but only the top 3 are used for naming /author/series/title/
var publishYear = null
var subtitle = null
// If Title is of format 1999 - Title, then use 1999 as publish year
var publishYearMatch = title.match(/^([0-9]{4}) - (.+)/)
if (publishYearMatch && publishYearMatch.length > 2) {
@ -60,6 +133,8 @@ function getAudiobookDataFromFilepath(abRootPath, relpath, parseSubtitle = false
}
}
// Subtitle can be parsed from the title if user enabled
var subtitle = null
if (parseSubtitle && title.includes(' - ')) {
var splitOnSubtitle = title.split(' - ')
title = splitOnSubtitle.shift()
@ -72,71 +147,34 @@ function getAudiobookDataFromFilepath(abRootPath, relpath, parseSubtitle = false
subtitle,
series,
publishYear,
path, // relative audiobook path i.e. /Author Name/Book Name/..
fullPath: Path.join(abRootPath, path) // i.e. /audiobook/Author Name/Book Name/..
path: dir, // relative audiobook path i.e. /Author Name/Book Name/..
fullPath: Path.join(abRootPath, dir) // i.e. /audiobook/Author Name/Book Name/..
}
}
async function getAllAudiobookFileData(abRootPath, serverSettings = {}) {
var parseSubtitle = !!serverSettings.scannerParseSubtitle
var paths = await getPaths(abRootPath)
var audiobooks = {}
paths.files.forEach((filepath) => {
var relpath = Path.normalize(filepath).replace(abRootPath, '').slice(1)
var parsed = Path.parse(relpath)
var path = parsed.dir
if (!audiobooks[path]) {
var audiobookData = getAudiobookDataFromFilepath(abRootPath, relpath, parseSubtitle)
if (!audiobookData) return
audiobooks[path] = {
...audiobookData,
audioFiles: [],
otherFiles: []
}
}
var fileObj = {
filetype: getFileType(parsed.ext),
filename: parsed.base,
path: relpath,
fullPath: filepath,
ext: parsed.ext
}
if (fileObj.filetype === 'audio') {
audiobooks[path].audioFiles.push(fileObj)
} else {
audiobooks[path].otherFiles.push(fileObj)
}
})
return Object.values(audiobooks)
}
module.exports.getAllAudiobookFileData = getAllAudiobookFileData
async function getAudiobookFileData(abRootPath, audiobookPath, serverSettings = {}) {
var parseSubtitle = !!serverSettings.scannerParseSubtitle
var paths = await getPaths(audiobookPath)
var audiobook = null
var filepaths = paths.files
paths.files.forEach((filepath) => {
// Sort by least number of directories
filepaths.sort((a, b) => {
var pathsA = Path.dirname(a).split(Path.sep).length
var pathsB = Path.dirname(b).split(Path.sep).length
return pathsA - pathsB
})
var audiobookDir = Path.normalize(audiobookPath).replace(abRootPath, '').slice(1)
var audiobookData = getAudiobookDataFromDir(abRootPath, audiobookDir, parseSubtitle)
var audiobook = {
...audiobookData,
audioFiles: [],
otherFiles: []
}
filepaths.forEach((filepath) => {
var relpath = Path.normalize(filepath).replace(abRootPath, '').slice(1)
if (!audiobook) {
var audiobookData = getAudiobookDataFromFilepath(abRootPath, relpath, parseSubtitle)
if (!audiobookData) return
audiobook = {
...audiobookData,
audioFiles: [],
otherFiles: []
}
}
var extname = Path.extname(filepath)
var basename = Path.basename(filepath)
var fileObj = {