From 354e54a356ca485a25a51629ea5adfad78e4491f Mon Sep 17 00:00:00 2001 From: Fredrik Strand Oseberg Date: Mon, 27 Mar 2023 11:24:01 +0200 Subject: [PATCH] fix: count events instead of loading them in memory (#3382) Refactor project events to use count instead of loading the events in memory --- src/lib/db/event-store.ts | 29 +++++++++++++ src/lib/server-impl.ts | 2 +- src/lib/services/index.ts | 15 +++---- src/lib/services/project-service.ts | 41 +++++++++++-------- src/lib/types/stores/event-store.ts | 1 + .../e2e/services/project-service.e2e.test.ts | 2 + src/test/fixtures/fake-event-store.ts | 5 +++ 7 files changed, 67 insertions(+), 28 deletions(-) diff --git a/src/lib/db/event-store.ts b/src/lib/db/event-store.ts index 89913c871b..302079fb85 100644 --- a/src/lib/db/event-store.ts +++ b/src/lib/db/event-store.ts @@ -207,6 +207,35 @@ class EventStore implements IEventStore { } } + async queryCount(operations: IQueryOperations[]): Promise { + try { + let query: Knex.QueryBuilder = this.db.count().from(TABLE); + + operations.forEach((operation) => { + if (operation.op === 'where') { + query = this.where(query, operation.parameters); + } + + if (operation.op === 'forFeatures') { + query = this.forFeatures(query, operation.parameters); + } + + if (operation.op === 'beforeDate') { + query = this.beforeDate(query, operation.parameters); + } + + if (operation.op === 'betweenDate') { + query = this.betweenDate(query, operation.parameters); + } + }); + + const queryResult = await query.first(); + return parseInt(queryResult.count || 0); + } catch (e) { + return 0; + } + } + where( query: Knex.QueryBuilder, parameters: { [key: string]: string }, diff --git a/src/lib/server-impl.ts b/src/lib/server-impl.ts index 4453317f28..eb02bb0b25 100644 --- a/src/lib/server-impl.ts +++ b/src/lib/server-impl.ts @@ -43,7 +43,7 @@ async function createApp( const db = createDb(config); const stores = createStores(config, db); const services = createServices(stores, config, db); - scheduleServices(services, config); + scheduleServices(services); const metricsMonitor = createMetricsMonitor(); const unleashSession = sessionDb(config, db); diff --git a/src/lib/services/index.ts b/src/lib/services/index.ts index e14de8bf77..9b731b1e45 100644 --- a/src/lib/services/index.ts +++ b/src/lib/services/index.ts @@ -58,10 +58,7 @@ import { } from '../features/change-request-access-service/createChangeRequestAccessReadModel'; // TODO: will be moved to scheduler feature directory -export const scheduleServices = ( - services: IUnleashServices, - config: IUnleashConfig, -): void => { +export const scheduleServices = (services: IUnleashServices): void => { const { schedulerService, apiTokenService, @@ -94,12 +91,10 @@ export const scheduleServices = ( hoursToMilliseconds(24), ); - if (config.flagResolver.isEnabled('projectStatusApi')) { - schedulerService.schedule( - projectService.statusJob.bind(projectService), - hoursToMilliseconds(24), - ); - } + schedulerService.schedule( + projectService.statusJob.bind(projectService), + hoursToMilliseconds(24), + ); schedulerService.schedule( projectHealthService.setHealthRating.bind(projectHealthService), diff --git a/src/lib/services/project-service.ts b/src/lib/services/project-service.ts index 22650dc5ae..4e3553540f 100644 --- a/src/lib/services/project-service.ts +++ b/src/lib/services/project-service.ts @@ -35,6 +35,7 @@ import { ProjectUserRemovedEvent, ProjectUserUpdateRoleEvent, RoleName, + IFlagResolver, } from '../types'; import { IProjectQuery, @@ -114,6 +115,8 @@ export default class ProjectService { private projectStatsStore: IProjectStatsStore; + private flagResolver: IFlagResolver; + constructor( { projectStore, @@ -157,6 +160,7 @@ export default class ProjectService { this.groupService = groupService; this.projectStatsStore = projectStatsStore; this.logger = config.getLogger('services/project-service.js'); + this.flagResolver = config.flagResolver; } async getProjects( @@ -664,20 +668,24 @@ export default class ProjectService { } async statusJob(): Promise { - const projects = await this.store.getAll(); + if (this.flagResolver.isEnabled('projectStatusApi')) { + const projects = await this.store.getAll(); - const statusUpdates = await Promise.all( - projects.map((project) => this.getStatusUpdates(project.id)), - ); + const statusUpdates = await Promise.all( + projects.map((project) => this.getStatusUpdates(project.id)), + ); - await Promise.all( - statusUpdates.map((statusUpdate) => { - return this.projectStatsStore.updateProjectStats( - statusUpdate.projectId, - statusUpdate.updates, - ); - }), - ); + await Promise.all( + statusUpdates.map((statusUpdate) => { + return this.projectStatsStore.updateProjectStats( + statusUpdate.projectId, + statusUpdate.updates, + ); + }), + ); + } else { + this.logger.info('Project status API is disabled'); + } } async getStatusUpdates(projectId: string): Promise { @@ -726,7 +734,7 @@ export default class ProjectService { const [projectActivityCurrentWindow, projectActivityPastWindow] = await Promise.all([ - this.eventStore.query([ + this.eventStore.queryCount([ { op: 'where', parameters: { project: projectId } }, { op: 'beforeDate', @@ -736,7 +744,7 @@ export default class ProjectService { }, }, ]), - this.eventStore.query([ + this.eventStore.queryCount([ { op: 'where', parameters: { project: projectId } }, { op: 'betweenDate', @@ -792,9 +800,8 @@ export default class ProjectService { createdPastWindow: createdPastWindow.length, archivedCurrentWindow: archivedCurrentWindow.length, archivedPastWindow: archivedPastWindow.length, - projectActivityCurrentWindow: - projectActivityCurrentWindow.length, - projectActivityPastWindow: projectActivityPastWindow.length, + projectActivityCurrentWindow: projectActivityCurrentWindow, + projectActivityPastWindow: projectActivityPastWindow, projectMembersAddedCurrentWindow: projectMembersAddedCurrentWindow, }, diff --git a/src/lib/types/stores/event-store.ts b/src/lib/types/stores/event-store.ts index 23392e202d..ac53c6b887 100644 --- a/src/lib/types/stores/event-store.ts +++ b/src/lib/types/stores/event-store.ts @@ -15,4 +15,5 @@ export interface IEventStore searchEvents(search: SearchEventsSchema): Promise; getMaxRevisionId(currentMax?: number): Promise; query(operations: IQueryOperations[]): Promise; + queryCount(operations: IQueryOperations[]): Promise; } diff --git a/src/test/e2e/services/project-service.e2e.test.ts b/src/test/e2e/services/project-service.e2e.test.ts index 91a4d6cf6e..4a315988ef 100644 --- a/src/test/e2e/services/project-service.e2e.test.ts +++ b/src/test/e2e/services/project-service.e2e.test.ts @@ -1390,4 +1390,6 @@ test('should get correct amount of project members for current and past window', const result = await projectService.getStatusUpdates(project.id); expect(result.updates.projectMembersAddedCurrentWindow).toBe(5); + expect(result.updates.projectActivityCurrentWindow).toBe(6); + expect(result.updates.projectActivityPastWindow).toBe(0); }); diff --git a/src/test/fixtures/fake-event-store.ts b/src/test/fixtures/fake-event-store.ts index ef21300720..af694b289c 100644 --- a/src/test/fixtures/fake-event-store.ts +++ b/src/test/fixtures/fake-event-store.ts @@ -95,6 +95,11 @@ class FakeEventStore implements IEventStore { return []; } + async queryCount(operations: IQueryOperations[]): Promise { + if (operations) return 0; + return 0; + } + setMaxListeners(number: number): EventEmitter { return this.eventEmitter.setMaxListeners(number); }