feat: statistics for orphaned tokens (#7568)

Added metrics for orphaned tokens and modified `createTokenRowReducer` to exclude tokens in v1 format.

src/lib/__snapshots__/create-config.test.ts.snap

@@ -76,7 +76,7 @@ exports[`should create default config 1`] = `
   "flagResolver": FlagResolver {
     "experiments": {
       "adminTokenKillSwitch": false,
-      "allowOrphanedWildcardTokens": false,
+      "allowOrphanedWildcardTokens": true,
       "anonymiseEventLog": false,
       "anonymizeProjectOwners": false,
       "automatedActions": false,

src/lib/db/api-token-store.ts

@@ -43,7 +43,8 @@ const createTokenRowReducer =
             if (
                 !allowOrphanedWildcardTokens &&
                 !tokenRow.project &&
-                !tokenRow.secret.startsWith('*:') &&
+                tokenRow.secret.includes(':') && // Exclude v1 tokens
+                !tokenRow.secret.startsWith('*:') && // Exclude intentionally wildcard
                 (tokenRow.type === ApiTokenType.CLIENT ||
                     tokenRow.type === ApiTokenType.FRONTEND)
             ) {
@@ -270,4 +271,71 @@ export class ApiTokenStore implements IApiTokenStore {
             this.logger.error('Could not update lastSeen, error: ', err);
         }
     }
+
+    async countDeprecatedTokens(): Promise<{
+        orphanedTokens: number;
+        activeOrphanedTokens: number;
+        legacyTokens: number;
+        activeLegacyTokens: number;
+    }> {
+        const allLegacyCount = this.db<ITokenRow>(`${TABLE} as tokens`)
+            .where('tokens.secret', 'NOT LIKE', '%:%')
+            .count()
+            .first()
+            .then((res) => Number(res?.count) || 0);
+
+        const activeLegacyCount = this.db<ITokenRow>(`${TABLE} as tokens`)
+            .where('tokens.secret', 'NOT LIKE', '%:%')
+            .andWhereRaw("tokens.seen_at > NOW() - INTERVAL '3 MONTH'")
+            .count()
+            .first()
+            .then((res) => Number(res?.count) || 0);
+
+        const orphanedTokensQuery = this.db<ITokenRow>(`${TABLE} as tokens`)
+            .leftJoin(
+                `${API_LINK_TABLE} as token_project_link`,
+                'tokens.secret',
+                'token_project_link.secret',
+            )
+            .whereNull('token_project_link.project')
+            .andWhere('tokens.secret', 'NOT LIKE', '*:%') // Exclude intentionally wildcard tokens
+            .andWhere('tokens.secret', 'LIKE', '%:%') // Exclude legacy tokens
+            .andWhere((builder) => {
+                builder
+                    .where('tokens.type', ApiTokenType.CLIENT)
+                    .orWhere('tokens.type', ApiTokenType.FRONTEND);
+            });
+
+        const allOrphanedCount = orphanedTokensQuery
+            .clone()
+            .count()
+            .first()
+            .then((res) => Number(res?.count) || 0);
+
+        const activeOrphanedCount = orphanedTokensQuery
+            .clone()
+            .andWhereRaw("tokens.seen_at > NOW() - INTERVAL '3 MONTH'")
+            .count()
+            .first()
+            .then((res) => Number(res?.count) || 0);
+
+        const [
+            orphanedTokens,
+            activeOrphanedTokens,
+            legacyTokens,
+            activeLegacyTokens,
+        ] = await Promise.all([
+            allOrphanedCount,
+            activeOrphanedCount,
+            allLegacyCount,
+            activeLegacyCount,
+        ]);
+
+        return {
+            orphanedTokens,
+            activeOrphanedTokens,
+            legacyTokens,
+            activeLegacyTokens,
+        };
+    }
 }

src/lib/features/project/project-service.e2e.test.ts

@@ -82,7 +82,9 @@ beforeAll(async () => {
     const config = createTestConfig({
         getLogger,
         experimental: {
-            flags: {},
+            flags: {
+                cleanApiTokenWhenOrphaned: true,
+            },
         },
     });
     eventService = new EventService(stores, config);

src/lib/metrics.ts

@@ -321,6 +321,26 @@ export default class MetricsMonitor {
             labelNames: ['project_id'],
         });
 
+        const orphanedTokensTotal = createGauge({
+            name: 'orphaned_api_tokens_total',
+            help: 'Number of API tokens without a project',
+        });
+
+        const orphanedTokensActive = createGauge({
+            name: 'orphaned_api_tokens_active',
+            help: 'Number of API tokens without a project, last seen within 3 months',
+        });
+
+        const legacyTokensTotal = createGauge({
+            name: 'legacy_api_tokens_total',
+            help: 'Number of API tokens with v1 format',
+        });
+
+        const legacyTokensActive = createGauge({
+            name: 'legacy_api_tokens_active',
+            help: 'Number of API tokens with v1 format, last seen within 3 months',
+        });
+
         async function collectStaticCounters() {
             try {
                 const stats = await instanceStatsService.getStats();
@@ -333,6 +353,7 @@ export default class MetricsMonitor {
                     stageDurationByProject,
                     largestProjectEnvironments,
                     largestFeatureEnvironments,
+                    deprecatedTokens,
                 ] = await Promise.all([
                     stores.featureStrategiesReadModel.getMaxFeatureStrategies(),
                     stores.featureStrategiesReadModel.getMaxFeatureEnvironmentStrategies(),
@@ -346,6 +367,7 @@ export default class MetricsMonitor {
                     stores.largestResourcesReadModel.getLargestFeatureEnvironments(
                         1,
                     ),
+                    stores.apiTokenStore.countDeprecatedTokens(),
                 ]);
 
                 featureFlagsTotal.reset();
@@ -394,6 +416,18 @@ export default class MetricsMonitor {
                     apiTokens.labels({ type }).set(value);
                 }
 
+                orphanedTokensTotal.reset();
+                orphanedTokensTotal.set(deprecatedTokens.orphanedTokens);
+
+                orphanedTokensActive.reset();
+                orphanedTokensActive.set(deprecatedTokens.activeOrphanedTokens);
+
+                legacyTokensTotal.reset();
+                legacyTokensTotal.set(deprecatedTokens.legacyTokens);
+
+                legacyTokensActive.reset();
+                legacyTokensActive.set(deprecatedTokens.activeLegacyTokens);
+
                 if (maxEnvironmentStrategies) {
                     maxFeatureEnvironmentStrategies.reset();
                     maxFeatureEnvironmentStrategies

src/lib/types/experimental.ts

@@ -305,7 +305,7 @@ const flags: IFlags = {
     ),
     allowOrphanedWildcardTokens: parseEnvVarBoolean(
         process.env.UNLEASH_ORPHANED_TOKENS_KILL_SWITCH,
-        false,
+        true,
     ),
     extendedMetrics: parseEnvVarBoolean(
         process.env.UNLEASH_EXPERIMENTAL_EXTENDED_METRICS,

src/lib/types/stores/api-token-store.ts

@@ -8,4 +8,10 @@ export interface IApiTokenStore extends Store<IApiToken, string> {
     markSeenAt(secrets: string[]): Promise<void>;
     count(): Promise<number>;
     countByType(): Promise<Map<string, number>>;
+    countDeprecatedTokens(): Promise<{
+        orphanedTokens: number;
+        activeOrphanedTokens: number;
+        legacyTokens: number;
+        activeLegacyTokens: number;
+    }>;
 }

src/test/e2e/api/client/feature.token.deleted.project.e2e.test.ts

@@ -23,7 +23,17 @@ const feature3 = 'f3.p2.token.access';
 
 beforeAll(async () => {
     db = await dbInit('feature_api_api_access_client_deletion', getLogger);
-    app = await setupAppWithAuth(db.stores, {}, db.rawDatabase);
+    app = await setupAppWithAuth(
+        db.stores,
+        {
+            experimental: {
+                flags: {
+                    cleanApiTokenWhenOrphaned: true,
+                },
+            },
+        },
+        db.rawDatabase,
+    );
     apiTokenService = app.services.apiTokenService;
 
     const { featureToggleServiceV2, environmentService } = app.services;

src/test/e2e/helpers/database-init.ts

@@ -35,6 +35,8 @@ async function resetDatabase(knex) {
         knex.table('tag_types').del(),
         knex.table('addons').del(),
         knex.table('users').del(),
+        knex.table('api_tokens').del(),
+        knex.table('api_token_project').del(),
         knex
             .table('reset_tokens')
             .del(),

src/test/e2e/stores/api-token-store.e2e.test.ts

@@ -11,6 +11,10 @@ beforeAll(async () => {
     stores = db.stores;
 });
 
+afterEach(async () => {
+    await db.reset();
+});
+
 afterAll(async () => {
     await db.destroy();
 });
@@ -35,3 +39,146 @@ test('get token returns the token when exists', async () => {
     expect(foundToken.tokenName).toBe(newToken.tokenName);
     expect(foundToken.type).toBe(newToken.type);
 });
+
+describe('count deprecated tokens', () => {
+    test('should return 0 if there is no legacy or orphaned tokens', async () => {
+        await stores.projectStore.create({
+            id: 'test',
+            name: 'test',
+        });
+        await stores.apiTokenStore.insert({
+            secret: '*:*.be44368985f7fb3237c584ef86f3d6bdada42ddbd63a019d26955178',
+            environment: 'default',
+            type: ApiTokenType.ADMIN,
+            projects: [],
+            tokenName: 'admin-token',
+        });
+        await stores.apiTokenStore.insert({
+            secret: 'default:development.be44368985f7fb3237c584ef86f3d6bdada42ddbd63a019d26955178',
+            environment: 'default',
+            type: ApiTokenType.CLIENT,
+            projects: ['default'],
+            tokenName: 'client-token',
+        });
+        await stores.apiTokenStore.insert({
+            secret: '*:development.be44368985f7fb3237c584ef86f3d6bdada42ddbd63a019d26955178',
+            environment: 'default',
+            type: ApiTokenType.CLIENT,
+            projects: [],
+            tokenName: 'client-wildcard-token',
+        });
+        await stores.apiTokenStore.insert({
+            secret: '[]:production.3d6bdada42ddbd63a019d26955178be44368985f7fb3237c584ef86f',
+            environment: 'default',
+            type: ApiTokenType.FRONTEND,
+            projects: ['default', 'test'],
+            tokenName: 'frontend-token',
+        });
+
+        const deprecatedTokens =
+            await stores.apiTokenStore.countDeprecatedTokens();
+
+        expect(deprecatedTokens).toEqual({
+            activeLegacyTokens: 0,
+            activeOrphanedTokens: 0,
+            legacyTokens: 0,
+            orphanedTokens: 0,
+        });
+    });
+
+    test('should return 1 for legacy tokens', async () => {
+        await stores.apiTokenStore.insert({
+            secret: 'be44368985f7fb3237c584ef86f3d6bdada42ddbd63a019d26955178',
+            environment: 'default',
+            type: ApiTokenType.ADMIN,
+            projects: [],
+            tokenName: 'admin-test-token',
+        });
+
+        const deprecatedTokens =
+            await stores.apiTokenStore.countDeprecatedTokens();
+
+        expect(deprecatedTokens).toEqual({
+            activeLegacyTokens: 0,
+            activeOrphanedTokens: 0,
+            legacyTokens: 1,
+            orphanedTokens: 0,
+        });
+    });
+
+    test('should return 1 for orphaned tokens', async () => {
+        await stores.apiTokenStore.insert({
+            secret: 'deleted-project:development.be44368985f7fb3237c584ef86f3d6bdada42ddbd63a019d26955178',
+            environment: 'default',
+            type: ApiTokenType.CLIENT,
+            projects: [],
+            tokenName: 'admin-test-token',
+        });
+
+        const deprecatedTokens =
+            await stores.apiTokenStore.countDeprecatedTokens();
+
+        expect(deprecatedTokens).toEqual({
+            activeLegacyTokens: 0,
+            activeOrphanedTokens: 0,
+            legacyTokens: 0,
+            orphanedTokens: 1,
+        });
+    });
+
+    test('should not count wildcard tokens as orphaned', async () => {
+        await stores.apiTokenStore.insert({
+            secret: '*:*.be44368985f7fb3237c584ef86f3d6bdada42ddbd63a019d26955178',
+            environment: 'default',
+            type: ApiTokenType.CLIENT,
+            projects: [],
+            tokenName: 'client-test-token',
+        });
+
+        const deprecatedTokens =
+            await stores.apiTokenStore.countDeprecatedTokens();
+
+        expect(deprecatedTokens).toEqual({
+            activeLegacyTokens: 0,
+            activeOrphanedTokens: 0,
+            legacyTokens: 0,
+            orphanedTokens: 0,
+        });
+    });
+
+    test('should count active tokens based on seen_at', async () => {
+        const legacyTokenSecret =
+            'be44368985f7fb3237c584ef86f3d6bdada42ddbd63a019d26955178';
+        const orphanedTokenSecret =
+            '[]:production.be44368985f7fb3237c584ef86f3d6bdada42ddbd63a019d26955178';
+        await stores.apiTokenStore.insert({
+            secret: legacyTokenSecret,
+            environment: 'default',
+            type: ApiTokenType.ADMIN,
+            projects: [],
+            tokenName: 'admin-test-token',
+        });
+        await stores.apiTokenStore.insert({
+            secret: orphanedTokenSecret,
+            environment: 'default',
+            type: ApiTokenType.FRONTEND,
+            projects: [],
+            tokenName: 'frontend-test-token',
+        });
+
+        await stores.apiTokenStore.markSeenAt([
+            legacyTokenSecret,
+            orphanedTokenSecret,
+        ]);
+
+        const deprecatedTokens =
+            await stores.apiTokenStore.countDeprecatedTokens();
+
+        expect(deprecatedTokens).toEqual({
+            activeLegacyTokens: 1,
+            activeOrphanedTokens: 1,
+            legacyTokens: 1,
+            orphanedTokens: 1,
+        });
+    });
+});

src/test/fixtures/fake-api-token-store.ts

@@ -78,4 +78,18 @@ export default class FakeApiTokenStore
         t.expiresAt = expiresAt;
         return t;
     }
+
+    async countDeprecatedTokens(): Promise<{
+        orphanedTokens: number;
+        activeOrphanedTokens: number;
+        legacyTokens: number;
+        activeLegacyTokens: number;
+    }> {
+        return {
+            orphanedTokens: 0,
+            activeOrphanedTokens: 0,
+            legacyTokens: 0,
+            activeLegacyTokens: 0,
+        };
+    }
 }