From 30073d527aeec0038a899b365a77cb853bac2da6 Mon Sep 17 00:00:00 2001 From: Simon Hornby Date: Thu, 4 Jul 2024 08:51:27 +0200 Subject: [PATCH] feat: extended SDK metrics (#7527) This adds an extended metrics format to the metrics ingested by Unleash and sent by running SDKs in the wild. Notably, we don't store this information anywhere new in this PR, this is just streamed out to Victoria metrics - the point of this project is insight, not analysis. Two things to look out for in this PR: - I've chosen to take extend the registration event and also send that when we receive metrics. This means that the new data is received on startup and on heartbeat. This takes us in the direction of collapsing these two calls into one at a later point - I've wrapped the existing metrics events in some "type safety", it ain't much because we have 0 type safety on the event emitter so this also has some if checks that look funny in TS that actually check if the data shape is correct. Existing tests that check this are more or less preserved --- .../__snapshots__/create-config.test.ts.snap | 1 + .../client-metrics/metrics-service-v2.ts | 20 ++++++++- .../metrics/instance/instance-service.ts | 19 +++++++- src/lib/metrics.test.ts | 27 +++++++----- src/lib/metrics.ts | 43 ++++++++++++++++--- .../openapi/spec/client-application-schema.ts | 24 +++++++++++ src/lib/openapi/spec/client-metrics-schema.ts | 34 ++++++++++++++- src/lib/types/experimental.ts | 7 ++- src/lib/types/model.ts | 17 ++++++++ src/server-dev.ts | 1 + 10 files changed, 170 insertions(+), 23 deletions(-) diff --git a/src/lib/__snapshots__/create-config.test.ts.snap b/src/lib/__snapshots__/create-config.test.ts.snap index f12556a309..614dcce0d9 100644 --- a/src/lib/__snapshots__/create-config.test.ts.snap +++ b/src/lib/__snapshots__/create-config.test.ts.snap @@ -100,6 +100,7 @@ exports[`should create default config 1`] = ` "enableLicenseChecker": false, "encryptEmails": false, "estimateTrafficDataCost": false, + "extendedMetrics": false, "extendedUsageMetrics": false, "featureLifecycle": false, "featureSearchFeedback": { diff --git a/src/lib/features/metrics/client-metrics/metrics-service-v2.ts b/src/lib/features/metrics/client-metrics/metrics-service-v2.ts index a753c3d76b..c002ef3fbe 100644 --- a/src/lib/features/metrics/client-metrics/metrics-service-v2.ts +++ b/src/lib/features/metrics/client-metrics/metrics-service-v2.ts @@ -4,7 +4,7 @@ import { type IFlagResolver, type IUnleashConfig, } from '../../../types'; -import type { IUnleashStores } from '../../../types'; +import type { ISdkHeartbeat, IUnleashStores } from '../../../types'; import type { ToggleMetricsSummary } from '../../../types/models/metrics'; import type { IClientMetricsEnv, @@ -12,7 +12,7 @@ import type { } from './client-metrics-store-v2-type'; import { clientMetricsSchema } from '../shared/schema'; import { compareAsc } from 'date-fns'; -import { CLIENT_METRICS } from '../../../types/events'; +import { CLIENT_METRICS, CLIENT_REGISTER } from '../../../types/events'; import ApiUser, { type IApiUser } from '../../../types/api-user'; import { ALL } from '../../../types/models/api-token'; import type { IUser } from '../../../types/user'; @@ -157,6 +157,22 @@ export default class ClientMetricsServiceV2 { `Got ${toggleNames.length} (${validatedToggleNames.length} valid) metrics from ${clientIp}`, ); + if (data.sdkVersion) { + const [sdkName, sdkVersion] = data.sdkVersion.split(':'); + const heartbeatEvent: ISdkHeartbeat = { + sdkName, + sdkVersion, + metadata: { + platformName: data.platformName, + platformVersion: data.platformVersion, + yggdrasilVersion: data.yggdrasilVersion, + specVersion: data.specVersion, + }, + }; + + this.config.eventBus.emit(CLIENT_REGISTER, heartbeatEvent); + } + if (validatedToggleNames.length > 0) { const clientMetrics: IClientMetricsEnv[] = validatedToggleNames.map( (name) => ({ diff --git a/src/lib/features/metrics/instance/instance-service.ts b/src/lib/features/metrics/instance/instance-service.ts index 315d3a16e4..30ff24ce14 100644 --- a/src/lib/features/metrics/instance/instance-service.ts +++ b/src/lib/features/metrics/instance/instance-service.ts @@ -12,7 +12,7 @@ import type { import type { IFeatureToggleStore } from '../../feature-toggle/types/feature-toggle-store-type'; import type { IStrategyStore } from '../../../types/stores/strategy-store'; import type { IClientInstanceStore } from '../../../types/stores/client-instance-store'; -import type { IClientApp } from '../../../types/model'; +import type { IClientApp, ISdkHeartbeat } from '../../../types/model'; import { clientRegisterSchema } from '../shared/schema'; import type { IClientMetricsStoreV2 } from '../client-metrics/client-metrics-store-v2-type'; @@ -105,7 +105,22 @@ export default class ClientInstanceService { value.clientIp = clientIp; value.createdBy = SYSTEM_USER.username!; this.seenClients[this.clientKey(value)] = value; - this.eventStore.emit(CLIENT_REGISTER, value); + + if (value.sdkVersion && value.sdkVersion.indexOf(':') > -1) { + const [sdkName, sdkVersion] = value.sdkVersion.split(':'); + const heartbeatEvent: ISdkHeartbeat = { + sdkName, + sdkVersion, + metadata: { + platformName: data.platformName, + platformVersion: data.platformVersion, + yggdrasilVersion: data.yggdrasilVersion, + specVersion: data.specVersion, + }, + }; + + this.eventStore.emit(CLIENT_REGISTER, heartbeatEvent); + } } async announceUnannounced(): Promise { diff --git a/src/lib/metrics.test.ts b/src/lib/metrics.test.ts index 94ef576511..a4020332bb 100644 --- a/src/lib/metrics.test.ts +++ b/src/lib/metrics.test.ts @@ -239,40 +239,47 @@ test('Should collect metrics for database', async () => { test('Should collect metrics for client sdk versions', async () => { eventStore.emit(CLIENT_REGISTER, { - sdkVersion: 'unleash-client-node:3.2.5', + sdkName: 'unleash-client-node', + sdkVersion: '3.2.5', }); eventStore.emit(CLIENT_REGISTER, { - sdkVersion: 'unleash-client-node:3.2.5', + sdkName: 'unleash-client-node', + sdkVersion: '3.2.5', }); eventStore.emit(CLIENT_REGISTER, { - sdkVersion: 'unleash-client-node:3.2.5', + sdkName: 'unleash-client-node', + sdkVersion: '3.2.5', }); eventStore.emit(CLIENT_REGISTER, { - sdkVersion: 'unleash-client-java:5.0.0', + sdkName: 'unleash-client-java', + sdkVersion: '5.0.0', }); eventStore.emit(CLIENT_REGISTER, { - sdkVersion: 'unleash-client-java:5.0.0', + sdkName: 'unleash-client-java', + sdkVersion: '5.0.0', }); eventStore.emit(CLIENT_REGISTER, { - sdkVersion: 'unleash-client-java:5.0.0', + sdkName: 'unleash-client-java', + sdkVersion: '5.0.0', }); const metrics = await prometheusRegister.getSingleMetricAsString( 'client_sdk_versions', ); expect(metrics).toMatch( - /client_sdk_versions\{sdk_name="unleash-client-node",sdk_version="3\.2\.5"\} 3/, + /client_sdk_versions\{sdk_name="unleash-client-node",sdk_version="3\.2\.5"\,platformName=\"not-set\",platformVersion=\"not-set\",yggdrasilVersion=\"not-set\",specVersion=\"not-set\"} 3/, ); expect(metrics).toMatch( - /client_sdk_versions\{sdk_name="unleash-client-java",sdk_version="5\.0\.0"\} 3/, + /client_sdk_versions\{sdk_name="unleash-client-java",sdk_version="5\.0\.0"\,platformName=\"not-set\",platformVersion=\"not-set\",yggdrasilVersion=\"not-set\",specVersion=\"not-set\"} 3/, ); eventStore.emit(CLIENT_REGISTER, { - sdkVersion: 'unleash-client-node:3.2.5', + sdkName: 'unleash-client-node', + sdkVersion: '3.2.5', }); const newmetrics = await prometheusRegister.getSingleMetricAsString( 'client_sdk_versions', ); expect(newmetrics).toMatch( - /client_sdk_versions\{sdk_name="unleash-client-node",sdk_version="3\.2\.5"\} 4/, + /client_sdk_versions\{sdk_name="unleash-client-node",sdk_version="3\.2\.5"\,platformName=\"not-set\",platformVersion=\"not-set\",yggdrasilVersion=\"not-set\",specVersion=\"not-set\"} 4/, ); }); diff --git a/src/lib/metrics.ts b/src/lib/metrics.ts index 18e1ea04b1..cae55b2624 100644 --- a/src/lib/metrics.ts +++ b/src/lib/metrics.ts @@ -24,7 +24,7 @@ import type { IUnleashConfig } from './types/option'; import type { ISettingStore, IUnleashStores } from './types/stores'; import { hoursToMilliseconds, minutesToMilliseconds } from 'date-fns'; import type { InstanceStatsService } from './features/instance-stats/instance-stats-service'; -import type { IEnvironment } from './types'; +import type { IEnvironment, ISdkHeartbeat } from './types'; import { createCounter, createGauge, @@ -51,6 +51,7 @@ export default class MetricsMonitor { } const { eventStore, environmentStore } = stores; + const { flagResolver } = config; const cachedEnvironments: () => Promise = memoizee( async () => environmentStore.getAll(), @@ -244,7 +245,14 @@ export default class MetricsMonitor { const clientSdkVersionUsage = createCounter({ name: 'client_sdk_versions', help: 'Which sdk versions are being used', - labelNames: ['sdk_name', 'sdk_version'], + labelNames: [ + 'sdk_name', + 'sdk_version', + 'platformName', + 'platformVersion', + 'yggdrasilVersion', + 'specVersion', + ], }); const productionChanges30 = createGauge({ @@ -784,15 +792,36 @@ export default class MetricsMonitor { } }); - eventStore.on(CLIENT_REGISTER, (m) => { - if (m.sdkVersion && m.sdkVersion.indexOf(':') > -1) { - const [sdkName, sdkVersion] = m.sdkVersion.split(':'); + eventStore.on(CLIENT_REGISTER, (heartbeatEvent: ISdkHeartbeat) => { + if (!heartbeatEvent.sdkName || !heartbeatEvent.sdkVersion) { + return; + } + + if (flagResolver.isEnabled('extendedMetrics')) { clientSdkVersionUsage.increment({ - sdk_name: sdkName, - sdk_version: sdkVersion, + sdk_name: heartbeatEvent.sdkName, + sdk_version: heartbeatEvent.sdkVersion, + platformName: + heartbeatEvent.metadata?.platformName ?? 'not-set', + platformVersion: + heartbeatEvent.metadata?.platformVersion ?? 'not-set', + yggdrasilVersion: + heartbeatEvent.metadata?.yggdrasilVersion ?? 'not-set', + specVersion: + heartbeatEvent.metadata?.specVersion ?? 'not-set', + }); + } else { + clientSdkVersionUsage.increment({ + sdk_name: heartbeatEvent.sdkName, + sdk_version: heartbeatEvent.sdkVersion, + platformName: 'not-set', + platformVersion: 'not-set', + yggdrasilVersion: 'not-set', + specVersion: 'not-set', }); } }); + eventStore.on(PROJECT_ENVIRONMENT_REMOVED, ({ project }) => { projectEnvironmentsDisabled.increment({ project_id: project }); }); diff --git a/src/lib/openapi/spec/client-application-schema.ts b/src/lib/openapi/spec/client-application-schema.ts index df73d8e827..b7e4f3d0dc 100644 --- a/src/lib/openapi/spec/client-application-schema.ts +++ b/src/lib/openapi/spec/client-application-schema.ts @@ -30,6 +30,30 @@ export const clientApplicationSchema = { type: 'string', example: 'development', }, + platformName: { + description: + 'The platform the application is running on. For languages that compile to binaries, this can be omitted', + type: 'string', + example: '.NET Core', + }, + platformVersion: { + description: + 'The version of the platform the application is running on. Languages that compile to binaries, this is expected to be the compiler version used to assemble the binary.', + type: 'string', + example: '3.1', + }, + yggdrasilVersion: { + description: + 'The semantic version of the Yggdrasil engine used by the client. If the client is using a native engine this can be omitted.', + type: 'string', + example: '1.0.0', + }, + specVersion: { + description: + 'The version of the Unleash client specification the client supports', + type: 'string', + example: '3.0.0', + }, interval: { type: 'number', description: diff --git a/src/lib/openapi/spec/client-metrics-schema.ts b/src/lib/openapi/spec/client-metrics-schema.ts index c41f073284..1c2d864291 100644 --- a/src/lib/openapi/spec/client-metrics-schema.ts +++ b/src/lib/openapi/spec/client-metrics-schema.ts @@ -21,9 +21,41 @@ export const clientMetricsSchema = { example: 'application-name-dacb1234', }, environment: { - description: 'Which environment the application is running in', + description: + 'Which environment the application is running in. This property was deprecated in v5. This can be determined by the API key calling this endpoint.', type: 'string', example: 'development', + deprecated: true, + }, + sdkVersion: { + type: 'string', + description: + 'An SDK version identifier. Usually formatted as "unleash-client-:"', + example: 'unleash-client-java:7.0.0', + }, + platformName: { + description: + 'The platform the application is running on. For languages that compile to binaries, this can be omitted', + type: 'string', + example: '.NET Core', + }, + platformVersion: { + description: + 'The version of the platform the application is running on. Languages that compile to binaries, this is expected to be the compiler version used to assemble the binary.', + type: 'string', + example: '3.1', + }, + yggdrasilVersion: { + description: + 'The semantic version of the Yggdrasil engine used by the client. If the client is using a native engine this can be omitted.', + type: 'string', + example: '1.0.0', + }, + specVersion: { + description: + 'The version of the Unleash client specification the client supports', + type: 'string', + example: '3.0.0', }, bucket: { type: 'object', diff --git a/src/lib/types/experimental.ts b/src/lib/types/experimental.ts index 2cd77e1604..4d2097d30a 100644 --- a/src/lib/types/experimental.ts +++ b/src/lib/types/experimental.ts @@ -62,7 +62,8 @@ export type IFlagKey = | 'commandBarUI' | 'flagCreator' | 'anonymizeProjectOwners' - | 'resourceLimits'; + | 'resourceLimits' + | 'extendedMetrics'; export type IFlags = Partial<{ [key in IFlagKey]: boolean | Variant }>; @@ -299,6 +300,10 @@ const flags: IFlags = { process.env.UNLEASH_EXPERIMENTAL_RESOURCE_LIMITS, false, ), + extendedMetrics: parseEnvVarBoolean( + process.env.UNLEASH_EXPERIMENTAL_EXTENDED_METRICS, + false, + ), }; export const defaultExperimentalOptions: IExperimentalOptions = { diff --git a/src/lib/types/model.ts b/src/lib/types/model.ts index e3529d04a9..d66fc87863 100644 --- a/src/lib/types/model.ts +++ b/src/lib/types/model.ts @@ -451,6 +451,10 @@ export interface IClientApp { icon?: string; description?: string; color?: string; + platformName?: string; + platformVersion?: string; + yggdrasilVersion?: string; + specVersion?: string; } export interface IAppFeature { @@ -598,3 +602,16 @@ export interface IUserAccessOverview { rootRole: string; groupProjects: string[]; } + +export interface ISdkHeartbeat { + sdkVersion: string; + sdkName: string; + metadata: ISdkHeartbeatMetadata; +} + +export interface ISdkHeartbeatMetadata { + platformName?: string; + platformVersion?: string; + yggdrasilVersion?: string; + specVersion?: string; +} diff --git a/src/server-dev.ts b/src/server-dev.ts index 9d8638748b..6dd12fcf6c 100644 --- a/src/server-dev.ts +++ b/src/server-dev.ts @@ -55,6 +55,7 @@ process.nextTick(async () => { commandBarUI: true, flagCreator: true, resourceLimits: true, + extendedMetrics: true, }, }, authentication: {