mirror of
https://github.com/Unleash/unleash.git
synced 2025-02-23 00:22:19 +01:00
chore: limit the amount of unannounced events we announce (#4845)
## About the changes
When the events table is large we might be doing a full table scan
searching for unannounced events. We spotted it due to a performance
alert and confirmed in AWS performance insights

The proposal is to limit this operation to 500 events (rule of thumb)
per round
f82ae354eb/src/lib/services/index.ts (L141-L147)
and also ignore the events older than a day (because it seems
reasonable)
## Discussion points
**Idea**: split the `events` table into `recent_events` and
`historical_events`. Recent can be anything from a day/week/month. This
would help with recurrent queries that rely on recent data from the
event's table such as optimal 304 calculation or event this scheduled
task that sends unannounced events.
This commit is contained in:
parent
87d9497be9
commit
e2ffbee468
@ -1,6 +1,8 @@
|
||||
import knex from 'knex';
|
||||
import EventStore from './event-store';
|
||||
import getLogger from '../../test/fixtures/no-logger';
|
||||
import { subHours, formatRFC3339 } from 'date-fns';
|
||||
import dbInit from '../../test/e2e/helpers/database-init';
|
||||
|
||||
beforeAll(() => {
|
||||
getLogger.setMuteError(true);
|
||||
@ -28,3 +30,38 @@ test('Trying to get events by name if db fails should yield empty list', async (
|
||||
expect(events).toBeTruthy();
|
||||
expect(events.length).toBe(0);
|
||||
});
|
||||
|
||||
test.each([
|
||||
{
|
||||
createdAt: formatRFC3339(subHours(new Date(), 1)),
|
||||
expectedCount: 1,
|
||||
},
|
||||
{
|
||||
createdAt: formatRFC3339(subHours(new Date(), 23)),
|
||||
expectedCount: 1,
|
||||
},
|
||||
{
|
||||
createdAt: formatRFC3339(subHours(new Date(), 25)),
|
||||
expectedCount: 0,
|
||||
},
|
||||
])(
|
||||
'Find unnanounced events is capped to last 24hs',
|
||||
async ({ createdAt, expectedCount }) => {
|
||||
const db = await dbInit('events_test', getLogger);
|
||||
const type = 'application-created' as const;
|
||||
const insertQuery = db.rawDatabase('events');
|
||||
await insertQuery
|
||||
.insert({
|
||||
type,
|
||||
created_at: createdAt,
|
||||
created_by: 'a test',
|
||||
data: { name: 'test', createdAt },
|
||||
})
|
||||
.returning(['id']);
|
||||
|
||||
const store = new EventStore(db.rawDatabase, getLogger);
|
||||
const events = await store.setUnannouncedToAnnounced();
|
||||
expect(events).toBeTruthy();
|
||||
expect(events.length).toBe(expectedCount);
|
||||
},
|
||||
);
|
||||
|
@ -12,6 +12,7 @@ import { sharedEventEmitter } from '../util/anyEventEmitter';
|
||||
import { Db } from './db';
|
||||
import { Knex } from 'knex';
|
||||
import EventEmitter from 'events';
|
||||
import { subDays } from 'date-fns';
|
||||
|
||||
const EVENT_COLUMNS = [
|
||||
'id',
|
||||
@ -407,12 +408,14 @@ class EventStore implements IEventStore {
|
||||
return this.eventEmitter.off(eventName, listener);
|
||||
}
|
||||
|
||||
private async setUnannouncedToAnnounced(): Promise<IEvent[]> {
|
||||
async setUnannouncedToAnnounced(): Promise<IEvent[]> {
|
||||
const rows = await this.db(TABLE)
|
||||
.update({ announced: true })
|
||||
.where('announced', false)
|
||||
.whereNotNull('announced')
|
||||
.returning(EVENT_COLUMNS);
|
||||
.where('created_at', '>', subDays(Date.now(), 1))
|
||||
.returning(EVENT_COLUMNS)
|
||||
.limit(500);
|
||||
|
||||
return rows.map(this.rowToEvent);
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user