feat: project applications paging backend (#6312)

frontend/src/component/common/MultiActionButton/MultiActionButton.tsx

@@ -1,6 +1,4 @@
-import React, { FC, useContext } from 'react';
+import React, { FC } from 'react';
-import { useRequiredPathParam } from 'hooks/useRequiredPathParam';
-import { useChangeRequest } from 'hooks/api/getters/useChangeRequest/useChangeRequest';
 
 import {
     ClickAwayListener,

src/lib/db/client-applications-store.ts

@@ -2,12 +2,14 @@ import EventEmitter from 'events';
 import NotFoundError from '../error/notfound-error';
 import {
     IClientApplication,
+    IClientApplications,
+    IClientApplicationsSearchParams,
     IClientApplicationsStore,
 } from '../types/stores/client-applications-store';
 import { Logger, LogProvider } from '../logger';
-import { IApplicationQuery } from '../types/query';
 import { Db } from './db';
 import { IApplicationOverview } from '../features/metrics/instance/models';
+import { applySearchFilters } from '../features/feature-search/search-utils';
 
 const COLUMNS = [
     'app_name',
@@ -64,7 +66,12 @@ const reduceRows = (rows: any[]): IClientApplication[] => {
                 ...mapRow(row),
                 usage:
                     project && environment
-                        ? [{ project, environments: [environment] }]
+                        ? [
+                              {
+                                  project,
+                                  environments: [environment],
+                              },
+                          ]
                         : [],
             };
         }
@@ -173,38 +180,62 @@ export default class ClientApplicationsStore
         return this.db(TABLE).where('app_name', appName).del();
     }
 
-    /**
+    async getApplications(
-     * Could also be done in SQL:
+        params: IClientApplicationsSearchParams,
-     * (not sure if it is faster though)
+    ): Promise<IClientApplications> {
-     *
+        const { limit, offset, sortOrder = 'asc', searchParams } = params;
-     * SELECT app_name from (
+        const validatedSortOrder =
-     *   SELECT app_name, json_array_elements(strategies)::text as strategyName from client_strategies
+            sortOrder === 'asc' || sortOrder === 'desc' ? sortOrder : 'asc';
-     *   ) as foo
-     * WHERE foo.strategyName = '"other"';
-     */
-    async getAppsForStrategy(
-        query: IApplicationQuery,
-    ): Promise<IClientApplication[]> {
-        const rows = await this.db
-            .select([
-                ...COLUMNS.map((column) => `${TABLE}.${column}`),
-                'project',
-                'environment',
-            ])
-            .from(TABLE)
-            .leftJoin(
-                TABLE_USAGE,
-                `${TABLE_USAGE}.app_name`,
-                `${TABLE}.app_name`,
-            );
-        const apps = reduceRows(rows);
 
-        if (query.strategyName) {
+        const query = this.db
-            return apps.filter((app) =>
+            .with('applications', (qb) => {
-                app.strategies.includes(query.strategyName),
+                applySearchFilters(qb, searchParams, [
-            );
+                    'client_applications.app_name',
+                ]);
+                qb.select([
+                    ...COLUMNS.map((column) => `${TABLE}.${column}`),
+                    'project',
+                    'environment',
+                    this.db.raw(
+                        `DENSE_RANK() OVER (ORDER BY client_applications.app_name ${validatedSortOrder}) AS rank`,
+                    ),
+                ])
+                    .from(TABLE)
+                    .leftJoin(
+                        TABLE_USAGE,
+                        `${TABLE_USAGE}.app_name`,
+                        `${TABLE}.app_name`,
+                    );
+            })
+            .with(
+                'final_ranks',
+                this.db.raw(
+                    'select row_number() over (order by min(rank)) as final_rank from applications group by app_name',
+                ),
+            )
+            .with(
+                'total',
+                this.db.raw('select count(*) as total from final_ranks'),
+            )
+            .select('*')
+            .from('applications')
+            .joinRaw('CROSS JOIN total')
+            .whereBetween('rank', [offset + 1, offset + limit]);
+
+        const rows = await query;
+
+        if (rows.length !== 0) {
+            const applications = reduceRows(rows);
+            return {
+                applications,
+                total: Number(rows[0].total) || 0,
+            };
         }
-        return apps;
+
+        return {
+            applications: [],
+            total: 0,
+        };
     }
 
     async getUnannounced(): Promise<IClientApplication[]> {

src/lib/features/feature-search/feature-search-controller.ts

@@ -19,6 +19,7 @@ import {
     FeatureSearchQueryParameters,
     featureSearchQueryParameters,
 } from '../../openapi/spec/feature-search-query-parameters';
+import { normalizeQueryParams } from './search-utils';
 
 const PATH = '/features';
 
@@ -83,17 +84,21 @@ export default class FeatureSearchController extends Controller {
             createdAt,
             state,
             status,
-            offset,
-            limit = '50',
-            sortOrder,
-            sortBy,
             favoritesFirst,
         } = req.query;
         const userId = req.user.id;
-        const normalizedQuery = query
+        const {
-            ?.split(',')
+            normalizedQuery,
-            .map((query) => query.trim())
+            normalizedSortBy,
-            .filter((query) => query);
+            normalizedSortOrder,
+            normalizedOffset,
+            normalizedLimit,
+        } = normalizeQueryParams(req.query, {
+            limitDefault: 50,
+            maxLimit: 100,
+            sortByDefault: 'createdAt',
+        });
+
         const normalizedStatus = status
             ?.map((tag) => tag.split(':'))
             .filter(
@@ -101,12 +106,6 @@ export default class FeatureSearchController extends Controller {
                     tag.length === 2 &&
                     ['enabled', 'disabled'].includes(tag[1]),
             );
-        const normalizedLimit =
-            Number(limit) > 0 && Number(limit) <= 100 ? Number(limit) : 25;
-        const normalizedOffset = Number(offset) > 0 ? Number(offset) : 0;
-        const normalizedSortBy: string = sortBy ? sortBy : 'createdAt';
-        const normalizedSortOrder =
-            sortOrder === 'asc' || sortOrder === 'desc' ? sortOrder : 'asc';
         const normalizedFavoritesFirst = favoritesFirst === 'true';
         const { features, total } = await this.featureSearchService.search({
             searchParams: normalizedQuery,

src/lib/features/feature-search/search-utils.ts

@@ -1,6 +1,13 @@
 import { Knex } from 'knex';
 import { IQueryParam } from '../feature-toggle/types/feature-toggle-strategies-store-type';
 
+export interface NormalizeParamsDefaults {
+    limitDefault: number;
+    maxLimit?: number; // Optional because you might not always want to enforce a max limit
+    sortByDefault: string;
+    typeDefault?: string; // Optional field for type, not required for every call
+}
+
 export const applySearchFilters = (
     qb: Knex.QueryBuilder,
     searchParams: string[] | undefined,
@@ -45,3 +52,41 @@ export const applyGenericQueryParams = (
         }
     });
 };
+
+export const normalizeQueryParams = (
+    params,
+    defaults: NormalizeParamsDefaults,
+) => {
+    const {
+        query,
+        offset,
+        limit = defaults.limitDefault,
+        sortOrder,
+        sortBy = defaults.sortByDefault,
+    } = params;
+
+    const normalizedQuery = query
+        ?.split(',')
+        .map((query) => query.trim())
+        .filter((query) => query);
+
+    const maxLimit = defaults.maxLimit || 1000;
+    const normalizedLimit =
+        Number(limit) > 0 && Number(limit) <= maxLimit
+            ? Number(limit)
+            : defaults.limitDefault;
+
+    const normalizedOffset = Number(offset) > 0 ? Number(offset) : 0;
+
+    const normalizedSortBy = sortBy;
+    const normalizedSortOrder =
+        sortOrder === 'asc' || sortOrder === 'desc' ? sortOrder : 'asc';
+
+    return {
+        normalizedQuery,
+        normalizedLimit,
+        normalizedOffset,
+        normalizedSortBy,
+        normalizedSortOrder,
+    };
+};

src/lib/features/metrics/instance/instance-service.ts

@@ -5,12 +5,13 @@ import { IUnleashConfig } from '../../../types/option';
 import { IEventStore } from '../../../types/stores/event-store';
 import {
     IClientApplication,
+    IClientApplications,
+    IClientApplicationsSearchParams,
     IClientApplicationsStore,
 } from '../../../types/stores/client-applications-store';
 import { IFeatureToggleStore } from '../../feature-toggle/types/feature-toggle-store-type';
 import { IStrategyStore } from '../../../types/stores/strategy-store';
 import { IClientInstanceStore } from '../../../types/stores/client-instance-store';
-import { IApplicationQuery } from '../../../types/query';
 import { IClientApp } from '../../../types/model';
 import { clientRegisterSchema } from '../shared/schema';
 
@@ -155,28 +156,31 @@ export default class ClientInstanceService {
     }
 
     async getApplications(
-        query: IApplicationQuery,
+        query: IClientApplicationsSearchParams,
         userId: number,
-    ): Promise<IClientApplication[]> {
+    ): Promise<IClientApplications> {
         const applications =
-            await this.clientApplicationsStore.getAppsForStrategy(query);
+            await this.clientApplicationsStore.getApplications(query);
         const accessibleProjects =
             await this.privateProjectChecker.getUserAccessibleProjects(userId);
         if (accessibleProjects.mode === 'all') {
             return applications;
         } else {
-            return applications.map((application) => {
+            return {
-                return {
+                applications: applications.applications.map((application) => {
-                    ...application,
+                    return {
-                    usage: application.usage?.filter(
+                        ...application,
-                        (usageItem) =>
+                        usage: application.usage?.filter(
-                            usageItem.project === ALL_PROJECTS ||
+                            (usageItem) =>
-                            accessibleProjects.projects.includes(
+                                usageItem.project === ALL_PROJECTS ||
-                                usageItem.project,
+                                accessibleProjects.projects.includes(
-                            ),
+                                    usageItem.project,
-                    ),
+                                ),
-                };
+                        ),
-            });
+                    };
+                }),
+                total: applications.total,
+            };
         }
     }
 

src/lib/features/project/project-controller.ts

@@ -39,6 +39,7 @@ import {
 } from '../../openapi/spec/project-applications-schema';
 import { NotFoundError } from '../../error';
 import { projectApplicationsQueryParameters } from '../../openapi/spec/project-applications-query-parameters';
+import { normalizeQueryParams } from '../feature-search/search-utils';
 
 export default class ProjectController extends Controller {
     private projectService: ProjectService;
@@ -272,21 +273,19 @@ export default class ProjectController extends Controller {
             throw new NotFoundError();
         }
 
-        const { query, offset, limit = '50', sortOrder, sortBy } = req.query;
-
         const { projectId } = req.params;
 
-        const normalizedQuery = query
+        const {
-            ?.split(',')
+            normalizedQuery,
-            .map((query) => query.trim())
+            normalizedSortBy,
-            .filter((query) => query);
+            normalizedSortOrder,
-
+            normalizedOffset,
-        const normalizedLimit =
+            normalizedLimit,
-            Number(limit) > 0 && Number(limit) <= 100 ? Number(limit) : 25;
+        } = normalizeQueryParams(req.query, {
-        const normalizedOffset = Number(offset) > 0 ? Number(offset) : 0;
+            limitDefault: 50,
-        const normalizedSortBy: string = sortBy ? sortBy : 'appName';
+            maxLimit: 100,
-        const normalizedSortOrder =
+            sortByDefault: 'appName',
-            sortOrder === 'asc' || sortOrder === 'desc' ? sortOrder : 'asc';
+        });
 
         const applications = await this.projectService.getApplications({
             searchParams: normalizedQuery,

src/lib/routes/admin-api/metrics.ts

@@ -24,6 +24,7 @@ import {
 } from '../../openapi/spec/application-overview-schema';
 import { OpenApiService } from '../../services';
 import { applicationsQueryParameters } from '../../openapi/spec/applications-query-parameters';
+import { normalizeQueryParams } from '../../features/feature-search/search-utils';
 
 class MetricsController extends Controller {
     private logger: Logger;
@@ -162,7 +163,9 @@ class MetricsController extends Controller {
     }
 
     async deleteApplication(
-        req: Request<{ appName: string }>,
+        req: Request<{
+            appName: string;
+        }>,
         res: Response,
     ): Promise<void> {
         const { appName } = req.params;
@@ -172,7 +175,13 @@ class MetricsController extends Controller {
     }
 
     async createApplication(
-        req: Request<{ appName: string }, unknown, CreateApplicationSchema>,
+        req: Request<
+            {
+                appName: string;
+            },
+            unknown,
+            CreateApplicationSchema
+        >,
         res: Response,
     ): Promise<void> {
         const input = {
@@ -188,15 +197,29 @@ class MetricsController extends Controller {
         res: Response<ApplicationsSchema>,
     ): Promise<void> {
         const { user } = req;
-        const query = req.query.strategyName
+        const {
-            ? { strategyName: req.query.strategyName as string }
+            normalizedQuery,
-            : {};
+            normalizedSortBy,
+            normalizedSortOrder,
+            normalizedOffset,
+            normalizedLimit,
+        } = normalizeQueryParams(req.query, {
+            limitDefault: 1000,
+            maxLimit: 1000,
+            sortByDefault: 'appName',
+        });
+
         const applications = await this.clientInstanceService.getApplications(
-            query,
+            {
+                searchParams: normalizedQuery,
+                offset: normalizedOffset,
+                limit: normalizedLimit,
+                sortBy: normalizedSortBy,
+                sortOrder: normalizedSortOrder,
+            },
             extractUserIdFromUser(user),
         );
-        // todo: change to total with pagination later
+        res.json(applications);
-        res.json({ applications, total: applications.length });
     }
 
     async getApplication(
@@ -209,6 +232,7 @@ class MetricsController extends Controller {
             await this.clientInstanceService.getApplication(appName);
         res.json(appDetails);
     }
+
     async getApplicationOverview(
         req: Request,
         res: Response<ApplicationOverviewSchema>,
@@ -228,4 +252,5 @@ class MetricsController extends Controller {
         );
     }
 }
+
 export default MetricsController;

src/lib/types/stores/client-applications-store.ts

@@ -1,5 +1,4 @@
 import { Store } from './store';
-import { IApplicationQuery } from '../query';
 import { IApplicationOverview } from '../../features/metrics/instance/models';
 
 export interface IClientApplicationUsage {
@@ -22,11 +21,26 @@ export interface IClientApplication {
     usage?: IClientApplicationUsage[];
 }
 
+export interface IClientApplications {
+    applications: IClientApplication[];
+    total: number;
+}
+
+export interface IClientApplicationsSearchParams {
+    searchParams?: string[];
+    offset: number;
+    limit: number;
+    sortBy: string;
+    sortOrder: 'asc' | 'desc';
+}
+
 export interface IClientApplicationsStore
     extends Store<IClientApplication, string> {
     upsert(details: Partial<IClientApplication>): Promise<void>;
     bulkUpsert(details: Partial<IClientApplication>[]): Promise<void>;
-    getAppsForStrategy(query: IApplicationQuery): Promise<IClientApplication[]>;
+    getApplications(
+        params: IClientApplicationsSearchParams,
+    ): Promise<IClientApplications>;
     getUnannounced(): Promise<IClientApplication[]>;
     setUnannouncedToAnnounced(): Promise<IClientApplication[]>;
     getApplicationOverview(appName: string): Promise<IApplicationOverview>;

src/test/fixtures/fake-client-applications-store.ts

@@ -1,9 +1,10 @@
 import {
     IClientApplication,
+    IClientApplications,
+    IClientApplicationsSearchParams,
     IClientApplicationsStore,
 } from '../../lib/types/stores/client-applications-store';
 import NotFoundError from '../../lib/error/notfound-error';
-import { IApplicationQuery } from '../../lib/types/query';
 import { IApplicationOverview } from '../../lib/features/metrics/instance/models';
 
 export default class FakeClientApplicationsStore
@@ -55,15 +56,13 @@ export default class FakeClientApplicationsStore
         return this.get(appName);
     }
 
-    async getAppsForStrategy(
+    async getApplications(
-        query: IApplicationQuery,
+        query: IClientApplicationsSearchParams,
-    ): Promise<IClientApplication[]> {
+    ): Promise<IClientApplications> {
-        if (query.strategyName) {
+        return {
-            return this.apps.filter((a) =>
+            applications: this.apps,
-                a.strategies.includes(query.strategyName),
+            total: this.apps.length,
-            );
+        };
-        }
-        return this.apps;
     }
 
     async getUnannounced(): Promise<IClientApplication[]> {