chore: keep latest integration events for each integration configuration (#7652)

https://linear.app/unleash/issue/2-2469/keep-the-latest-event-for-each-integration-configuration

This makes it so we keep the latest event for each integration
configuration, along with the previous logic of keeping the latest 100
events of the last 2 hours.

This should be a cheap nice-to-have, since now we can always know what
the latest integration event looked like for each integration
configuration. This will tie-in nicely with the next task of making the
latest integration event state visible in the integration card.

Also improved the clarity of the auto-deletion explanation in the modal.

frontend/src/component/integrations/IntegrationEventsModal/IntegrationEventsModal.tsx

@@ -105,8 +105,10 @@ export const IntegrationEventsModal = ({
                     </StyledHeaderRow>
                     <StyledHeaderRow>
                         <StyledSubtitle>
-                            All events older than the last 100 or older than the
-                            past 2 hours will be automatically deleted.
+                            Only the most recent event for each integration and
+                            the last 100 events from the past 2 hours will be
+                            kept. All other events will be automatically
+                            deleted.
                         </StyledSubtitle>
                     </StyledHeaderRow>
                 </StyledHeader>

src/lib/features/integration-events/integration-events-store.ts

@@ -26,26 +26,45 @@ export class IntegrationEventsStore extends CRUDStore<
         limit: number,
         offset: number,
     ): Promise<IntegrationEventSchema[]> {
+        const endTimer = this.timer('getPaginatedEvents');
+
         const rows = await this.db(this.tableName)
             .where('integration_id', id)
             .limit(limit)
             .offset(offset)
             .orderBy('id', 'desc');
 
+        endTimer();
+
         return rows.map(this.fromRow) as IntegrationEventSchema[];
     }
 
     async cleanUpEvents(): Promise<void> {
-        return this.db
+        const endTimer = this.timer('cleanUpEvents');
+
+        await this.db
             .with('latest_events', (qb) => {
                 qb.select('id')
                     .from(this.tableName)
                     .whereRaw(`created_at >= now() - INTERVAL '2 hours'`)
-                    .orderBy('created_at', 'desc')
+                    .orderBy('id', 'desc')
                     .limit(100);
             })
+            .with('latest_per_integration', (qb) => {
+                qb.select(this.db.raw('MAX(id) as id'))
+                    .from(this.tableName)
+                    .groupBy('integration_id');
+            })
             .from(this.tableName)
-            .whereNotIn('id', this.db.select('id').from('latest_events'))
+            .whereNotIn(
+                'id',
+                this.db
+                    .select('id')
+                    .from('latest_events')
+                    .union(this.db.select('id').from('latest_per_integration')),
+            )
             .delete();
+
+        endTimer();
     }
 }

src/lib/features/integration-events/integration-events.e2e.test.ts

@@ -5,6 +5,7 @@ import {
 } from '../../../test/e2e/helpers/test-helper';
 import getLogger from '../../../test/fixtures/no-logger';
 import { TEST_AUDIT_USER } from '../../types';
+import type { IAddonDto } from '../../types/stores/addon-store';
 import type { IntegrationEventsService } from './integration-events-service';
 import type { IntegrationEventWriteModel } from './integration-events-store';
 
@@ -35,6 +36,15 @@ const EVENT_FAILED: IntegrationEventWriteModel = {
     stateDetails: 'Better Call Saul!',
 };
 
+const INTEGRATION: IAddonDto = {
+    provider: 'webhook',
+    enabled: true,
+    parameters: {
+        url: 'http://some-test-url',
+    },
+    events: ['feature-created'],
+};
+
 beforeAll(async () => {
     db = await dbInit('integration_events', getLogger);
     app = await setupAppWithCustomConfig(
@@ -55,14 +65,7 @@ beforeEach(async () => {
     await db.reset();
 
     const { id } = await app.services.addonService.createAddon(
-        {
-            provider: 'webhook',
-            enabled: true,
-            parameters: {
-                url: 'http://some-test-url',
-            },
-            events: ['feature-created'],
-        },
+        INTEGRATION,
         TEST_AUDIT_USER,
     );
 
@@ -193,6 +196,45 @@ test('clean up events, keeping the last 100 events', async () => {
     expect(events).toHaveLength(100);
 });
 
+test('clean up events, keeping the latest event for each integration', async () => {
+    const longTimeAgo = new Date('2000-01-01');
+
+    const { id: integrationId2 } = await app.services.addonService.createAddon(
+        INTEGRATION,
+        TEST_AUDIT_USER,
+    );
+
+    await insertPastEvent(getTestEventSuccess(), longTimeAgo);
+    await insertPastEvent(getTestEventFailed(), longTimeAgo);
+
+    await insertPastEvent(
+        { ...getTestEventSuccess(), integrationId: integrationId2 },
+        longTimeAgo,
+    );
+    await insertPastEvent(
+        { ...getTestEventFailed(), integrationId: integrationId2 },
+        longTimeAgo,
+    );
+
+    await integrationEventsService.cleanUpEvents();
+
+    const eventsIntegration1 =
+        await integrationEventsService.getPaginatedEvents(integrationId, 10, 0);
+
+    expect(eventsIntegration1).toHaveLength(1);
+    expect(eventsIntegration1[0].state).toBe('failed');
+
+    const eventsIntegration2 =
+        await integrationEventsService.getPaginatedEvents(
+            integrationId2,
+            10,
+            0,
+        );
+
+    expect(eventsIntegration2).toHaveLength(1);
+    expect(eventsIntegration2[0].state).toBe('failed');
+});
+
 test('return events from the API', async () => {
     await integrationEventsService.registerEvent(getTestEventSuccess());
     await integrationEventsService.registerEvent(getTestEventFailed());