Support accent-insensitive matching using the sqlean sqlite3 unicode extension

server/Database.js

@@ -207,6 +207,7 @@ class Database {
 
     try {
       await this.sequelize.authenticate()
+      await this.loadExtensions([process.env.SQLEAN_UNICODE_PATH])
       Logger.info(`[Database] Db connection was successful`)
       return true
     } catch (error) {
@@ -215,6 +216,30 @@ class Database {
     }
   }
 
+  async loadExtensions(extensions) {
+    // This is a hack to get the db connection for loading extensions.
+    // The proper way would be to use the 'afterConnect' hook, but that hook is never called for sqlite due to a bug in sequelize.
+    // See https://github.com/sequelize/sequelize/issues/12487
+    // This is not a public API and may break in the future.
+    const db = await this.sequelize.dialect.connectionManager.getConnection()
+    if (typeof db?.loadExtension !== 'function') throw new Error('Failed to get db connection for loading extensions')
+
+    for (const ext of extensions) {
+      Logger.info(`[Database] Loading extension ${ext}`)
+      await new Promise((resolve, reject) => {
+        db.loadExtension(ext, (err) => {
+          if (err) {
+            Logger.error(`[Database] Failed to load extension ${ext}`, err)
+            reject(err)
+            return
+          }
+          Logger.info(`[Database] Successfully loaded extension ${ext}`)
+          resolve()
+        })
+      })
+    }
+  }
+
   /**
    * Disconnect from db
    */
@@ -801,6 +826,23 @@ class Database {
       Logger.warn(`Removed ${badSessionsRemoved} sessions that were 3 seconds or less`)
     }
   }
+
+  normalize(value) {
+    return `lower(unaccent(${value}))`
+  }
+
+  async getNormalizedQuery(query) {
+    const escapedQuery = this.sequelize.escape(query)
+    const normalizedQuery = this.normalize(escapedQuery)
+    const normalizedQueryResult = await this.sequelize.query(`SELECT ${normalizedQuery} as normalized_query`)
+    return normalizedQueryResult[0][0].normalized_query
+  }
+
+  matchExpression(column, normalizedQuery) {
+    const normalizedPattern = this.sequelize.escape(`%${normalizedQuery}%`)
+    const normalizedColumn = this.normalize(column)
+    return `${normalizedColumn} LIKE ${normalizedPattern}`
+  }
 }
 
 module.exports = new Database()

server/utils/queries/authorFilters.js

@@ -60,12 +60,10 @@ module.exports = {
    * @returns {Promise<Object[]>} oldAuthor with numBooks
    */
   async search(libraryId, query, limit, offset) {
+    const matchAuthor = Database.matchExpression('name', query)
     const authors = await Database.authorModel.findAll({
       where: {
-        name: {
+        [Sequelize.Op.and]: [Sequelize.literal(matchAuthor), { libraryId }]
-          [Sequelize.Op.substring]: query
-        },
-        libraryId
       },
       attributes: {
         include: [[Sequelize.literal('(SELECT count(*) FROM bookAuthors ba WHERE ba.authorId = author.id)'), 'numBooks']]

server/utils/queries/libraryItemsBookFilters.js

@@ -975,21 +975,18 @@ module.exports = {
   async search(oldUser, oldLibrary, query, limit, offset) {
     const userPermissionBookWhere = this.getUserPermissionBookWhereQuery(oldUser)
 
+    const normalizedQuery = await Database.getNormalizedQuery(query)
+
+    const matchTitle = Database.matchExpression('title', normalizedQuery)
+    const matchSubtitle = Database.matchExpression('subtitle', normalizedQuery)
+
     // Search title, subtitle, asin, isbn
     const books = await Database.bookModel.findAll({
       where: [
         {
           [Sequelize.Op.or]: [
-            {
+            Sequelize.literal(matchTitle),
-              title: {
+            Sequelize.literal(matchSubtitle),
-                [Sequelize.Op.substring]: query
-              }
-            },
-            {
-              subtitle: {
-                [Sequelize.Op.substring]: query
-              }
-            },
             {
               asin: {
                 [Sequelize.Op.substring]: query
@@ -1044,11 +1041,12 @@ module.exports = {
       })
     }
 
+    const matchJsonValue = Database.matchExpression('json_each.value', normalizedQuery)
+
     // Search narrators
     const narratorMatches = []
-    const [narratorResults] = await Database.sequelize.query(`SELECT value, count(*) AS numBooks FROM books b, libraryItems li, json_each(b.narrators) WHERE json_valid(b.narrators) AND json_each.value LIKE :query AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value LIMIT :limit OFFSET :offset;`, {
+    const [narratorResults] = await Database.sequelize.query(`SELECT value, count(*) AS numBooks FROM books b, libraryItems li, json_each(b.narrators) WHERE json_valid(b.narrators) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value LIMIT :limit OFFSET :offset;`, {
       replacements: {
-        query: `%${query}%`,
         libraryId: oldLibrary.id,
         limit,
         offset
@@ -1064,9 +1062,8 @@ module.exports = {
 
     // Search tags
     const tagMatches = []
-    const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.tags) WHERE json_valid(b.tags) AND json_each.value LIKE :query AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
+    const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.tags) WHERE json_valid(b.tags) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
       replacements: {
-        query: `%${query}%`,
         libraryId: oldLibrary.id,
         limit,
         offset
@@ -1082,9 +1079,8 @@ module.exports = {
 
     // Search genres
     const genreMatches = []
-    const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.genres) WHERE json_valid(b.genres) AND json_each.value LIKE :query AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
+    const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM books b, libraryItems li, json_each(b.genres) WHERE json_valid(b.genres) AND ${matchJsonValue} AND b.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
       replacements: {
-        query: `%${query}%`,
         libraryId: oldLibrary.id,
         limit,
         offset
@@ -1099,12 +1095,15 @@ module.exports = {
     }
 
     // Search series
+    const matchName = Database.matchExpression('name', normalizedQuery)
     const allSeries = await Database.seriesModel.findAll({
       where: {
-        name: {
+        [Sequelize.Op.and]: [
-          [Sequelize.Op.substring]: query
+          Sequelize.literal(matchName),
-        },
+          {
             libraryId: oldLibrary.id
+          }
+        ]
       },
       replacements: userPermissionBookWhere.replacements,
       include: {
@@ -1137,7 +1136,7 @@ module.exports = {
     }
 
     // Search authors
-    const authorMatches = await authorFilters.search(oldLibrary.id, query, limit, offset)
+    const authorMatches = await authorFilters.search(oldLibrary.id, normalizedQuery, limit, offset)
 
     return {
       book: itemMatches,

server/utils/queries/libraryItemsPodcastFilters.js

@@ -313,21 +313,18 @@ module.exports = {
    */
   async search(oldUser, oldLibrary, query, limit, offset) {
     const userPermissionPodcastWhere = this.getUserPermissionPodcastWhereQuery(oldUser)
+
+    const normalizedQuery = await Database.getNormalizedQuery(query)
+    const matchTitle = Database.matchExpression('title', normalizedQuery)
+    const matchAuthor = Database.matchExpression('author', normalizedQuery)
+
     // Search title, author, itunesId, itunesArtistId
     const podcasts = await Database.podcastModel.findAll({
       where: [
         {
           [Sequelize.Op.or]: [
-            {
+            Sequelize.literal(matchTitle),
-              title: {
+            Sequelize.literal(matchAuthor),
-                [Sequelize.Op.substring]: query
-              }
-            },
-            {
-              author: {
-                [Sequelize.Op.substring]: query
-              }
-            },
             {
               itunesId: {
                 [Sequelize.Op.substring]: query
@@ -368,11 +365,12 @@ module.exports = {
       })
     }
 
+    const matchJsonValue = Database.matchExpression('json_each.value', normalizedQuery)
+
     // Search tags
     const tagMatches = []
-    const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.tags) WHERE json_valid(p.tags) AND json_each.value LIKE :query AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
+    const [tagResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.tags) WHERE json_valid(p.tags) AND ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
       replacements: {
-        query: `%${query}%`,
         libraryId: oldLibrary.id,
         limit,
         offset
@@ -388,9 +386,8 @@ module.exports = {
 
     // Search genres
     const genreMatches = []
-    const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.genres) WHERE json_valid(p.genres) AND json_each.value LIKE :query AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
+    const [genreResults] = await Database.sequelize.query(`SELECT value, count(*) AS numItems FROM podcasts p, libraryItems li, json_each(p.genres) WHERE json_valid(p.genres) AND ${matchJsonValue} AND p.id = li.mediaId AND li.libraryId = :libraryId GROUP BY value ORDER BY numItems DESC LIMIT :limit OFFSET :offset;`, {
       replacements: {
-        query: `%${query}%`,
         libraryId: oldLibrary.id,
         limit,
         offset