task: Make operations on the API Token store auditable. (#2531)

## About the changes
We need a way to have an audit log for operations made on Api Tokens.
These changes adds three new event types, API_TOKEN_CREATED,
API_TOKEN_UPDATED, API_TOKEN_DELETED and extends api-token-service to
store these to our event store to reflect the action being taken.

src/lib/db/api-token-store.ts

@@ -173,7 +173,10 @@ export class ApiTokenStore implements IApiTokenStore {
     }
 
     async get(key: string): Promise<IApiToken> {
-        const row = await this.makeTokenProjectQuery().where('secret', key);
+        const row = await this.makeTokenProjectQuery().where(
+            'tokens.secret',
+            key,
+        );
         return toTokens(row)[0];
     }
 

src/lib/routes/admin-api/api-token.ts

@@ -35,6 +35,7 @@ import {
 import { UpdateApiTokenSchema } from '../../openapi/spec/update-api-token-schema';
 import { emptyResponse } from '../../openapi/util/standard-responses';
 import { ProxyService } from '../../services/proxy-service';
+import { extractUsername } from '../../util';
 
 interface TokenParam {
     token: string;
@@ -159,7 +160,10 @@ export class ApiTokenController extends Controller {
         res: Response<ApiTokenSchema>,
     ): Promise<any> {
         const createToken = await createApiToken.validateAsync(req.body);
-        const token = await this.apiTokenService.createApiToken(createToken);
+        const token = await this.apiTokenService.createApiToken(
+            createToken,
+            extractUsername(req),
+        );
         this.openApiService.respondWithValidation(
             201,
             res,
@@ -181,7 +185,11 @@ export class ApiTokenController extends Controller {
             return res.status(400).send();
         }
 
-        await this.apiTokenService.updateExpiry(token, new Date(expiresAt));
+        await this.apiTokenService.updateExpiry(
+            token,
+            new Date(expiresAt),
+            extractUsername(req),
+        );
         return res.status(200).end();
     }
 
@@ -191,7 +199,7 @@ export class ApiTokenController extends Controller {
     ): Promise<void> {
         const { token } = req.params;
 
-        await this.apiTokenService.delete(token);
+        await this.apiTokenService.delete(token, extractUsername(req));
         this.proxyService.deleteClientForProxyToken(token);
         res.status(200).end();
     }

src/lib/services/api-token-service.test.ts

@@ -4,6 +4,13 @@ import { IUnleashConfig } from '../server-impl';
 import { ApiTokenType, IApiTokenCreate } from '../types/models/api-token';
 import FakeApiTokenStore from '../../test/fixtures/fake-api-token-store';
 import FakeEnvironmentStore from '../../test/fixtures/fake-environment-store';
+import FakeEventStore from '../../test/fixtures/fake-event-store';
+import {
+    API_TOKEN_CREATED,
+    API_TOKEN_DELETED,
+    API_TOKEN_UPDATED,
+} from '../types';
+import { addDays } from 'date-fns';
 
 test('Should init api token', async () => {
     const token = {
@@ -21,11 +28,15 @@ test('Should init api token', async () => {
     });
     const apiTokenStore = new FakeApiTokenStore();
     const environmentStore = new FakeEnvironmentStore();
+    const eventStore = new FakeEventStore();
     const insertCalled = new Promise((resolve) => {
         apiTokenStore.on('insert', resolve);
     });
 
-    new ApiTokenService({ apiTokenStore, environmentStore }, config);
+    new ApiTokenService(
+        { apiTokenStore, environmentStore, eventStore },
+        config,
+    );
 
     await insertCalled;
 
@@ -47,6 +58,7 @@ test("Shouldn't return frontend token when secret is undefined", async () => {
     const config: IUnleashConfig = createTestConfig({});
     const apiTokenStore = new FakeApiTokenStore();
     const environmentStore = new FakeEnvironmentStore();
+    const eventStore = new FakeEventStore();
 
     await environmentStore.create({
         name: 'default',
@@ -57,7 +69,7 @@ test("Shouldn't return frontend token when secret is undefined", async () => {
     });
 
     const apiTokenService = new ApiTokenService(
-        { apiTokenStore, environmentStore },
+        { apiTokenStore, environmentStore, eventStore },
         config,
     );
 
@@ -67,3 +79,60 @@ test("Shouldn't return frontend token when secret is undefined", async () => {
     expect(apiTokenService.getUserForToken(undefined)).toEqual(undefined);
     expect(apiTokenService.getUserForToken('')).toEqual(undefined);
 });
+
+test('Api token operations should all have events attached', async () => {
+    const token: IApiTokenCreate = {
+        environment: 'default',
+        projects: ['*'],
+        secret: '*:*:some-random-string',
+        type: ApiTokenType.FRONTEND,
+        username: 'front',
+        expiresAt: null,
+    };
+
+    const config: IUnleashConfig = createTestConfig({});
+    const apiTokenStore = new FakeApiTokenStore();
+    const environmentStore = new FakeEnvironmentStore();
+    const eventStore = new FakeEventStore();
+
+    await environmentStore.create({
+        name: 'default',
+        enabled: true,
+        protected: true,
+        type: 'test',
+        sortOrder: 1,
+    });
+
+    const apiTokenService = new ApiTokenService(
+        { apiTokenStore, environmentStore, eventStore },
+        config,
+    );
+    let saved = await apiTokenService.createApiTokenWithProjects(token);
+    let newExpiry = addDays(new Date(), 30);
+    await apiTokenService.updateExpiry(saved.secret, newExpiry, 'test');
+    await apiTokenService.delete(saved.secret, 'test');
+    const events = await eventStore.getEvents();
+    const createdApiTokenEvents = events.filter(
+        (e) => e.type === API_TOKEN_CREATED,
+    );
+    expect(createdApiTokenEvents).toHaveLength(1);
+    expect(createdApiTokenEvents[0].preData).toBeUndefined();
+    expect(createdApiTokenEvents[0].data.secret).toBeUndefined();
+
+    const updatedApiTokenEvents = events.filter(
+        (e) => e.type === API_TOKEN_UPDATED,
+    );
+    expect(updatedApiTokenEvents).toHaveLength(1);
+    expect(updatedApiTokenEvents[0].preData.expiresAt).toBeDefined();
+    expect(updatedApiTokenEvents[0].preData.secret).toBeUndefined();
+    expect(updatedApiTokenEvents[0].data.secret).toBeUndefined();
+    expect(updatedApiTokenEvents[0].data.expiresAt).toBe(newExpiry);
+
+    const deletedApiTokenEvents = events.filter(
+        (e) => e.type === API_TOKEN_DELETED,
+    );
+    expect(deletedApiTokenEvents).toHaveLength(1);
+    expect(deletedApiTokenEvents[0].data).toBeUndefined();
+    expect(deletedApiTokenEvents[0].preData).toBeDefined();
+    expect(deletedApiTokenEvents[0].preData.secret).toBeUndefined();
+});

src/lib/services/api-token-service.ts

@@ -1,7 +1,7 @@
 import crypto from 'crypto';
 import { Logger } from '../logger';
 import { ADMIN, CLIENT, FRONTEND } from '../types/permissions';
-import { IUnleashStores } from '../types/stores';
+import { IEventStore, IUnleashStores } from '../types/stores';
 import { IUnleashConfig } from '../types/option';
 import ApiUser from '../types/api-user';
 import {
@@ -20,6 +20,12 @@ import BadDataError from '../error/bad-data-error';
 import { minutesToMilliseconds } from 'date-fns';
 import { IEnvironmentStore } from 'lib/types/stores/environment-store';
 import { constantTimeCompare } from '../util/constantTimeCompare';
+import {
+    ApiTokenCreatedEvent,
+    ApiTokenDeletedEvent,
+    ApiTokenUpdatedEvent,
+} from '../types';
+import { omitKeys } from '../util';
 
 const resolveTokenPermissions = (tokenType: string) => {
     if (tokenType === ApiTokenType.ADMIN) {
@@ -48,14 +54,21 @@ export class ApiTokenService {
 
     private activeTokens: IApiToken[] = [];
 
+    private eventStore: IEventStore;
+
     constructor(
         {
             apiTokenStore,
             environmentStore,
-        }: Pick<IUnleashStores, 'apiTokenStore' | 'environmentStore'>,
+            eventStore,
+        }: Pick<
+            IUnleashStores,
+            'apiTokenStore' | 'environmentStore' | 'eventStore'
+        >,
         config: Pick<IUnleashConfig, 'getLogger' | 'authentication'>,
     ) {
         this.store = apiTokenStore;
+        this.eventStore = eventStore;
         this.environmentStore = environmentStore;
         this.logger = config.getLogger('/services/api-token-service.ts');
         this.fetchActiveTokens();
@@ -95,7 +108,7 @@ export class ApiTokenService {
         try {
             const createAll = tokens
                 .map(mapLegacyTokenWithSecret)
-                .map((t) => this.insertNewApiToken(t));
+                .map((t) => this.insertNewApiToken(t, 'init-api-tokens'));
             await Promise.all(createAll);
         } catch (e) {
             this.logger.error('Unable to create initial Admin API tokens');
@@ -140,12 +153,31 @@ export class ApiTokenService {
     public async updateExpiry(
         secret: string,
         expiresAt: Date,
+        updatedBy: string,
     ): Promise<IApiToken> {
-        return this.store.setExpiry(secret, expiresAt);
+        const previous = await this.store.get(secret);
+        const token = await this.store.setExpiry(secret, expiresAt);
+        await this.eventStore.store(
+            new ApiTokenUpdatedEvent({
+                createdBy: updatedBy,
+                previousToken: omitKeys(previous, 'secret'),
+                apiToken: omitKeys(token, 'secret'),
+            }),
+        );
+        return token;
     }
 
-    public async delete(secret: string): Promise<void> {
-        return this.store.delete(secret);
+    public async delete(secret: string, deletedBy: string): Promise<void> {
+        if (await this.store.exists(secret)) {
+            const token = await this.store.get(secret);
+            await this.store.delete(secret);
+            await this.eventStore.store(
+                new ApiTokenDeletedEvent({
+                    createdBy: deletedBy,
+                    apiToken: omitKeys(token, 'secret'),
+                }),
+            );
+        }
     }
 
     /**
@@ -153,13 +185,15 @@ export class ApiTokenService {
      */
     public async createApiToken(
         newToken: Omit<ILegacyApiTokenCreate, 'secret'>,
+        createdBy: string = 'unleash-system',
     ): Promise<IApiToken> {
         const token = mapLegacyToken(newToken);
-        return this.createApiTokenWithProjects(token);
+        return this.createApiTokenWithProjects(token, createdBy);
     }
 
     public async createApiTokenWithProjects(
         newToken: Omit<IApiTokenCreate, 'secret'>,
+        createdBy: string = 'unleash-system',
     ): Promise<IApiToken> {
         validateApiToken(newToken);
 
@@ -168,7 +202,7 @@ export class ApiTokenService {
 
         const secret = this.generateSecretKey(newToken);
         const createNewToken = { ...newToken, secret };
-        return this.insertNewApiToken(createNewToken);
+        return this.insertNewApiToken(createNewToken, createdBy);
     }
 
     // TODO: Remove this service method after embedded proxy has been released in
@@ -180,15 +214,22 @@ export class ApiTokenService {
 
         const secret = this.generateSecretKey(newToken);
         const createNewToken = { ...newToken, secret };
-        return this.insertNewApiToken(createNewToken);
+        return this.insertNewApiToken(createNewToken, 'system-migration');
     }
 
     private async insertNewApiToken(
         newApiToken: IApiTokenCreate,
+        createdBy: string,
     ): Promise<IApiToken> {
         try {
             const token = await this.store.insert(newApiToken);
             this.activeTokens.push(token);
+            await this.eventStore.store(
+                new ApiTokenCreatedEvent({
+                    createdBy,
+                    apiToken: omitKeys(token, 'secret'),
+                }),
+            );
             return token;
         } catch (error) {
             if (error.code === FOREIGN_KEY_VIOLATION) {

src/lib/types/events.ts

@@ -1,4 +1,5 @@
 import { FeatureToggle, IStrategyConfig, ITag, IVariant } from './model';
+import { IApiToken } from './models/api-token';
 
 export const APPLICATION_CREATED = 'application-created';
 
@@ -94,6 +95,10 @@ export const CHANGE_REQUEST_CANCELLED = 'change-request-cancelled';
 export const CHANGE_REQUEST_SENT_TO_REVIEW = 'change-request-sent-to-review';
 export const CHANGE_REQUEST_APPLIED = 'change-request-applied';
 
+export const API_TOKEN_CREATED = 'api-token-created';
+export const API_TOKEN_UPDATED = 'api-token-updated';
+export const API_TOKEN_DELETED = 'api-token-deleted';
+
 export interface IBaseEvent {
     type: string;
     createdBy: string;
@@ -604,3 +609,61 @@ export class PublicSignupTokenUserAddedEvent extends BaseEvent {
         this.data = eventData.data;
     }
 }
+
+export class ApiTokenCreatedEvent extends BaseEvent {
+    readonly data: any;
+
+    readonly environment: string;
+
+    readonly project: string;
+
+    constructor(eventData: {
+        createdBy: string;
+        apiToken: Omit<IApiToken, 'secret'>;
+    }) {
+        super(API_TOKEN_CREATED, eventData.createdBy);
+        this.data = eventData.apiToken;
+        this.environment = eventData.apiToken.environment;
+        this.project = eventData.apiToken.project;
+    }
+}
+
+export class ApiTokenDeletedEvent extends BaseEvent {
+    readonly preData: any;
+
+    readonly environment: string;
+
+    readonly project: string;
+
+    constructor(eventData: {
+        createdBy: string;
+        apiToken: Omit<IApiToken, 'secret'>;
+    }) {
+        super(API_TOKEN_DELETED, eventData.createdBy);
+        this.preData = eventData.apiToken;
+        this.environment = eventData.apiToken.environment;
+        this.project = eventData.apiToken.project;
+    }
+}
+
+export class ApiTokenUpdatedEvent extends BaseEvent {
+    readonly preData: any;
+
+    readonly data: any;
+
+    readonly environment: string;
+
+    readonly project: string;
+
+    constructor(eventData: {
+        createdBy: string;
+        previousToken: Omit<IApiToken, 'secret'>;
+        apiToken: Omit<IApiToken, 'secret'>;
+    }) {
+        super(API_TOKEN_UPDATED, eventData.createdBy);
+        this.preData = eventData.previousToken;
+        this.data = eventData.apiToken;
+        this.environment = eventData.apiToken.environment;
+        this.project = eventData.apiToken.project;
+    }
+}

src/test/e2e/services/api-token-service.e2e.test.ts

@@ -120,15 +120,18 @@ test('should update expiry of token', async () => {
     const time = new Date('2022-01-01');
     const newTime = new Date('2023-01-01');
 
-    const token = await apiTokenService.createApiToken({
-        username: 'default-client',
-        type: ApiTokenType.CLIENT,
-        expiresAt: time,
-        project: '*',
-        environment: DEFAULT_ENV,
-    });
+    const token = await apiTokenService.createApiToken(
+        {
+            username: 'default-client',
+            type: ApiTokenType.CLIENT,
+            expiresAt: time,
+            project: '*',
+            environment: DEFAULT_ENV,
+        },
+        'tester',
+    );
 
-    await apiTokenService.updateExpiry(token.secret, newTime);
+    await apiTokenService.updateExpiry(token.secret, newTime, 'tester');
 
     const [updatedToken] = await apiTokenService.getAllTokens();