1
0
mirror of https://github.com/Unleash/unleash.git synced 2025-02-23 00:22:19 +01:00

chore: limit the amount of unannounced events we announce (#4845)

## About the changes
When the events table is large we might be doing a full table scan
searching for unannounced events. We spotted it due to a performance
alert and confirmed in AWS performance insights

![image](https://github.com/Unleash/unleash/assets/455064/8e815fa3-7a1b-4453-881a-98a148eae119)

The proposal is to limit this operation to 500 events (rule of thumb)
per round
f82ae354eb/src/lib/services/index.ts (L141-L147)
and also ignore the events older than a day (because it seems
reasonable)


## Discussion points
**Idea**: split the `events` table into `recent_events` and
`historical_events`. Recent can be anything from a day/week/month. This
would help with recurrent queries that rely on recent data from the
event's table such as optimal 304 calculation or event this scheduled
task that sends unannounced events.
This commit is contained in:
Gastón Fournier 2023-09-27 15:38:00 +02:00 committed by GitHub
parent 87d9497be9
commit e2ffbee468
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 42 additions and 2 deletions

View File

@ -1,6 +1,8 @@
import knex from 'knex'; import knex from 'knex';
import EventStore from './event-store'; import EventStore from './event-store';
import getLogger from '../../test/fixtures/no-logger'; import getLogger from '../../test/fixtures/no-logger';
import { subHours, formatRFC3339 } from 'date-fns';
import dbInit from '../../test/e2e/helpers/database-init';
beforeAll(() => { beforeAll(() => {
getLogger.setMuteError(true); getLogger.setMuteError(true);
@ -28,3 +30,38 @@ test('Trying to get events by name if db fails should yield empty list', async (
expect(events).toBeTruthy(); expect(events).toBeTruthy();
expect(events.length).toBe(0); expect(events.length).toBe(0);
}); });
test.each([
{
createdAt: formatRFC3339(subHours(new Date(), 1)),
expectedCount: 1,
},
{
createdAt: formatRFC3339(subHours(new Date(), 23)),
expectedCount: 1,
},
{
createdAt: formatRFC3339(subHours(new Date(), 25)),
expectedCount: 0,
},
])(
'Find unnanounced events is capped to last 24hs',
async ({ createdAt, expectedCount }) => {
const db = await dbInit('events_test', getLogger);
const type = 'application-created' as const;
const insertQuery = db.rawDatabase('events');
await insertQuery
.insert({
type,
created_at: createdAt,
created_by: 'a test',
data: { name: 'test', createdAt },
})
.returning(['id']);
const store = new EventStore(db.rawDatabase, getLogger);
const events = await store.setUnannouncedToAnnounced();
expect(events).toBeTruthy();
expect(events.length).toBe(expectedCount);
},
);

View File

@ -12,6 +12,7 @@ import { sharedEventEmitter } from '../util/anyEventEmitter';
import { Db } from './db'; import { Db } from './db';
import { Knex } from 'knex'; import { Knex } from 'knex';
import EventEmitter from 'events'; import EventEmitter from 'events';
import { subDays } from 'date-fns';
const EVENT_COLUMNS = [ const EVENT_COLUMNS = [
'id', 'id',
@ -407,12 +408,14 @@ class EventStore implements IEventStore {
return this.eventEmitter.off(eventName, listener); return this.eventEmitter.off(eventName, listener);
} }
private async setUnannouncedToAnnounced(): Promise<IEvent[]> { async setUnannouncedToAnnounced(): Promise<IEvent[]> {
const rows = await this.db(TABLE) const rows = await this.db(TABLE)
.update({ announced: true }) .update({ announced: true })
.where('announced', false) .where('announced', false)
.whereNotNull('announced') .whereNotNull('announced')
.returning(EVENT_COLUMNS); .where('created_at', '>', subDays(Date.now(), 1))
.returning(EVENT_COLUMNS)
.limit(500);
return rows.map(this.rowToEvent); return rows.map(this.rowToEvent);
} }