1
0
mirror of https://github.com/Unleash/unleash.git synced 2024-12-22 19:07:54 +01:00

fix: count events instead of loading them in memory (#3382)

Refactor project events to use count instead of loading the events in
memory
This commit is contained in:
Fredrik Strand Oseberg 2023-03-27 11:24:01 +02:00 committed by GitHub
parent 30c1ebc3c3
commit 354e54a356
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 67 additions and 28 deletions

View File

@ -207,6 +207,35 @@ class EventStore implements IEventStore {
}
}
async queryCount(operations: IQueryOperations[]): Promise<number> {
try {
let query: Knex.QueryBuilder = this.db.count().from(TABLE);
operations.forEach((operation) => {
if (operation.op === 'where') {
query = this.where(query, operation.parameters);
}
if (operation.op === 'forFeatures') {
query = this.forFeatures(query, operation.parameters);
}
if (operation.op === 'beforeDate') {
query = this.beforeDate(query, operation.parameters);
}
if (operation.op === 'betweenDate') {
query = this.betweenDate(query, operation.parameters);
}
});
const queryResult = await query.first();
return parseInt(queryResult.count || 0);
} catch (e) {
return 0;
}
}
where(
query: Knex.QueryBuilder,
parameters: { [key: string]: string },

View File

@ -43,7 +43,7 @@ async function createApp(
const db = createDb(config);
const stores = createStores(config, db);
const services = createServices(stores, config, db);
scheduleServices(services, config);
scheduleServices(services);
const metricsMonitor = createMetricsMonitor();
const unleashSession = sessionDb(config, db);

View File

@ -58,10 +58,7 @@ import {
} from '../features/change-request-access-service/createChangeRequestAccessReadModel';
// TODO: will be moved to scheduler feature directory
export const scheduleServices = (
services: IUnleashServices,
config: IUnleashConfig,
): void => {
export const scheduleServices = (services: IUnleashServices): void => {
const {
schedulerService,
apiTokenService,
@ -94,12 +91,10 @@ export const scheduleServices = (
hoursToMilliseconds(24),
);
if (config.flagResolver.isEnabled('projectStatusApi')) {
schedulerService.schedule(
projectService.statusJob.bind(projectService),
hoursToMilliseconds(24),
);
}
schedulerService.schedule(
projectService.statusJob.bind(projectService),
hoursToMilliseconds(24),
);
schedulerService.schedule(
projectHealthService.setHealthRating.bind(projectHealthService),

View File

@ -35,6 +35,7 @@ import {
ProjectUserRemovedEvent,
ProjectUserUpdateRoleEvent,
RoleName,
IFlagResolver,
} from '../types';
import {
IProjectQuery,
@ -114,6 +115,8 @@ export default class ProjectService {
private projectStatsStore: IProjectStatsStore;
private flagResolver: IFlagResolver;
constructor(
{
projectStore,
@ -157,6 +160,7 @@ export default class ProjectService {
this.groupService = groupService;
this.projectStatsStore = projectStatsStore;
this.logger = config.getLogger('services/project-service.js');
this.flagResolver = config.flagResolver;
}
async getProjects(
@ -664,20 +668,24 @@ export default class ProjectService {
}
async statusJob(): Promise<void> {
const projects = await this.store.getAll();
if (this.flagResolver.isEnabled('projectStatusApi')) {
const projects = await this.store.getAll();
const statusUpdates = await Promise.all(
projects.map((project) => this.getStatusUpdates(project.id)),
);
const statusUpdates = await Promise.all(
projects.map((project) => this.getStatusUpdates(project.id)),
);
await Promise.all(
statusUpdates.map((statusUpdate) => {
return this.projectStatsStore.updateProjectStats(
statusUpdate.projectId,
statusUpdate.updates,
);
}),
);
await Promise.all(
statusUpdates.map((statusUpdate) => {
return this.projectStatsStore.updateProjectStats(
statusUpdate.projectId,
statusUpdate.updates,
);
}),
);
} else {
this.logger.info('Project status API is disabled');
}
}
async getStatusUpdates(projectId: string): Promise<ICalculateStatus> {
@ -726,7 +734,7 @@ export default class ProjectService {
const [projectActivityCurrentWindow, projectActivityPastWindow] =
await Promise.all([
this.eventStore.query([
this.eventStore.queryCount([
{ op: 'where', parameters: { project: projectId } },
{
op: 'beforeDate',
@ -736,7 +744,7 @@ export default class ProjectService {
},
},
]),
this.eventStore.query([
this.eventStore.queryCount([
{ op: 'where', parameters: { project: projectId } },
{
op: 'betweenDate',
@ -792,9 +800,8 @@ export default class ProjectService {
createdPastWindow: createdPastWindow.length,
archivedCurrentWindow: archivedCurrentWindow.length,
archivedPastWindow: archivedPastWindow.length,
projectActivityCurrentWindow:
projectActivityCurrentWindow.length,
projectActivityPastWindow: projectActivityPastWindow.length,
projectActivityCurrentWindow: projectActivityCurrentWindow,
projectActivityPastWindow: projectActivityPastWindow,
projectMembersAddedCurrentWindow:
projectMembersAddedCurrentWindow,
},

View File

@ -15,4 +15,5 @@ export interface IEventStore
searchEvents(search: SearchEventsSchema): Promise<IEvent[]>;
getMaxRevisionId(currentMax?: number): Promise<number>;
query(operations: IQueryOperations[]): Promise<IEvent[]>;
queryCount(operations: IQueryOperations[]): Promise<number>;
}

View File

@ -1390,4 +1390,6 @@ test('should get correct amount of project members for current and past window',
const result = await projectService.getStatusUpdates(project.id);
expect(result.updates.projectMembersAddedCurrentWindow).toBe(5);
expect(result.updates.projectActivityCurrentWindow).toBe(6);
expect(result.updates.projectActivityPastWindow).toBe(0);
});

View File

@ -95,6 +95,11 @@ class FakeEventStore implements IEventStore {
return [];
}
async queryCount(operations: IQueryOperations[]): Promise<number> {
if (operations) return 0;
return 0;
}
setMaxListeners(number: number): EventEmitter {
return this.eventEmitter.setMaxListeners(number);
}