mirror of
https://github.com/Unleash/unleash.git
synced 2025-10-04 11:17:02 +02:00
Instance stats should not rely on prometheus cause it can be disabled
This commit is contained in:
parent
5edd0f879f
commit
e7de20fc99
@ -10,8 +10,10 @@ import type { IClientInstanceStore } from '../../types';
|
|||||||
let instanceStatsService: InstanceStatsService;
|
let instanceStatsService: InstanceStatsService;
|
||||||
let versionService: VersionService;
|
let versionService: VersionService;
|
||||||
let clientInstanceStore: IClientInstanceStore;
|
let clientInstanceStore: IClientInstanceStore;
|
||||||
|
let updateMetrics: () => Promise<void>;
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
|
jest.clearAllMocks();
|
||||||
|
|
||||||
register.clear();
|
register.clear();
|
||||||
|
|
||||||
const config = createTestConfig();
|
const config = createTestConfig();
|
||||||
@ -31,13 +33,14 @@ beforeEach(() => {
|
|||||||
createFakeGetProductionChanges(),
|
createFakeGetProductionChanges(),
|
||||||
);
|
);
|
||||||
|
|
||||||
registerPrometheusMetrics(
|
const { collectDbMetrics } = registerPrometheusMetrics(
|
||||||
config,
|
config,
|
||||||
stores,
|
stores,
|
||||||
undefined as unknown as string,
|
undefined as unknown as string,
|
||||||
config.eventBus,
|
config.eventBus,
|
||||||
instanceStatsService,
|
instanceStatsService,
|
||||||
);
|
);
|
||||||
|
updateMetrics = collectDbMetrics;
|
||||||
|
|
||||||
jest.spyOn(clientInstanceStore, 'getDistinctApplicationsCount');
|
jest.spyOn(clientInstanceStore, 'getDistinctApplicationsCount');
|
||||||
jest.spyOn(instanceStatsService, 'getStats');
|
jest.spyOn(instanceStatsService, 'getStats');
|
||||||
@ -46,13 +49,12 @@ beforeEach(() => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
test('get snapshot should not call getStats', async () => {
|
test('get snapshot should not call getStats', async () => {
|
||||||
await instanceStatsService.dbMetrics.refreshDbMetrics();
|
await updateMetrics();
|
||||||
expect(
|
expect(
|
||||||
clientInstanceStore.getDistinctApplicationsCount,
|
clientInstanceStore.getDistinctApplicationsCount,
|
||||||
).toHaveBeenCalledTimes(3);
|
).toHaveBeenCalledTimes(3);
|
||||||
expect(instanceStatsService.getStats).toHaveBeenCalledTimes(0);
|
expect(instanceStatsService.getStats).toHaveBeenCalledTimes(0);
|
||||||
|
|
||||||
// subsequent calls to getStatsSnapshot don't call getStats
|
|
||||||
for (let i = 0; i < 3; i++) {
|
for (let i = 0; i < 3; i++) {
|
||||||
const { clientApps } = await instanceStatsService.getStats();
|
const { clientApps } = await instanceStatsService.getStats();
|
||||||
expect(clientApps).toStrictEqual([
|
expect(clientApps).toStrictEqual([
|
||||||
|
@ -29,7 +29,6 @@ import { CUSTOM_ROOT_ROLE_TYPE } from '../../util';
|
|||||||
import type { GetActiveUsers } from './getActiveUsers';
|
import type { GetActiveUsers } from './getActiveUsers';
|
||||||
import type { ProjectModeCount } from '../project/project-store';
|
import type { ProjectModeCount } from '../project/project-store';
|
||||||
import type { GetProductionChanges } from './getProductionChanges';
|
import type { GetProductionChanges } from './getProductionChanges';
|
||||||
import { DbMetricsMonitor } from '../../metrics-gauge';
|
|
||||||
|
|
||||||
export type TimeRange = 'allTime' | '30d' | '7d';
|
export type TimeRange = 'allTime' | '30d' | '7d';
|
||||||
|
|
||||||
@ -110,14 +109,12 @@ export class InstanceStatsService {
|
|||||||
|
|
||||||
private appCount?: Partial<{ [key in TimeRange]: number }>;
|
private appCount?: Partial<{ [key in TimeRange]: number }>;
|
||||||
|
|
||||||
private getActiveUsers: GetActiveUsers;
|
getActiveUsers: GetActiveUsers;
|
||||||
|
|
||||||
private getProductionChanges: GetProductionChanges;
|
getProductionChanges: GetProductionChanges;
|
||||||
|
|
||||||
private featureStrategiesReadModel: IFeatureStrategiesReadModel;
|
private featureStrategiesReadModel: IFeatureStrategiesReadModel;
|
||||||
|
|
||||||
dbMetrics: DbMetricsMonitor;
|
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
{
|
{
|
||||||
featureToggleStore,
|
featureToggleStore,
|
||||||
@ -181,20 +178,18 @@ export class InstanceStatsService {
|
|||||||
this.clientMetricsStore = clientMetricsStoreV2;
|
this.clientMetricsStore = clientMetricsStoreV2;
|
||||||
this.flagResolver = flagResolver;
|
this.flagResolver = flagResolver;
|
||||||
this.featureStrategiesReadModel = featureStrategiesReadModel;
|
this.featureStrategiesReadModel = featureStrategiesReadModel;
|
||||||
this.dbMetrics = new DbMetricsMonitor({ getLogger });
|
|
||||||
}
|
|
||||||
|
|
||||||
async fromPrometheus(
|
|
||||||
name: string,
|
|
||||||
labels?: Record<string, string | number>,
|
|
||||||
): Promise<number> {
|
|
||||||
return (await this.dbMetrics.findValue(name, labels)) ?? 0;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
getProjectModeCount(): Promise<ProjectModeCount[]> {
|
getProjectModeCount(): Promise<ProjectModeCount[]> {
|
||||||
return this.projectStore.getProjectModeCounts();
|
return this.projectStore.getProjectModeCounts();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
getToggleCount(): Promise<number> {
|
||||||
|
return this.featureToggleStore.count({
|
||||||
|
archived: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
getArchivedToggleCount(): Promise<number> {
|
getArchivedToggleCount(): Promise<number> {
|
||||||
return this.featureToggleStore.count({
|
return this.featureToggleStore.count({
|
||||||
archived: true,
|
archived: true,
|
||||||
@ -217,9 +212,6 @@ export class InstanceStatsService {
|
|||||||
return settings?.enabled || false;
|
return settings?.enabled || false;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* use getStatsSnapshot for low latency, sacrificing data-freshness
|
|
||||||
*/
|
|
||||||
async getStats(): Promise<InstanceStats> {
|
async getStats(): Promise<InstanceStats> {
|
||||||
const versionInfo = await this.versionService.getVersionInfo();
|
const versionInfo = await this.versionService.getVersionInfo();
|
||||||
const [
|
const [
|
||||||
@ -249,24 +241,24 @@ export class InstanceStatsService {
|
|||||||
maxConstraintValues,
|
maxConstraintValues,
|
||||||
maxConstraints,
|
maxConstraints,
|
||||||
] = await Promise.all([
|
] = await Promise.all([
|
||||||
this.fromPrometheus('feature_toggles_total'),
|
this.getToggleCount(),
|
||||||
this.getArchivedToggleCount(),
|
this.getArchivedToggleCount(),
|
||||||
this.userStore.count(),
|
this.getRegisteredUsers(),
|
||||||
this.userStore.countServiceAccounts(),
|
this.countServiceAccounts(),
|
||||||
this.apiTokenStore.countByType(),
|
this.countApiTokensByType(),
|
||||||
this.getActiveUsers(),
|
this.getActiveUsers(),
|
||||||
this.getProjectModeCount(),
|
this.getProjectModeCount(),
|
||||||
this.contextFieldStore.count(),
|
this.contextFieldCount(),
|
||||||
this.groupStore.count(),
|
this.groupCount(),
|
||||||
this.roleStore.count(),
|
this.roleCount(),
|
||||||
this.roleStore.filteredCount({ type: CUSTOM_ROOT_ROLE_TYPE }),
|
this.customRolesCount(),
|
||||||
this.roleStore.filteredCountInUse({ type: CUSTOM_ROOT_ROLE_TYPE }),
|
this.customRolesCountInUse(),
|
||||||
this.environmentStore.count(),
|
this.environmentCount(),
|
||||||
this.segmentStore.count(),
|
this.segmentCount(),
|
||||||
this.strategyStore.count(),
|
this.strategiesCount(),
|
||||||
this.hasSAML(),
|
this.hasSAML(),
|
||||||
this.hasOIDC(),
|
this.hasOIDC(),
|
||||||
this.clientAppCounts(),
|
this.appCount ? this.appCount : this.getLabeledAppCounts(),
|
||||||
this.eventStore.deprecatedFilteredCount({
|
this.eventStore.deprecatedFilteredCount({
|
||||||
type: FEATURES_EXPORTED,
|
type: FEATURES_EXPORTED,
|
||||||
}),
|
}),
|
||||||
@ -274,7 +266,7 @@ export class InstanceStatsService {
|
|||||||
type: FEATURES_IMPORTED,
|
type: FEATURES_IMPORTED,
|
||||||
}),
|
}),
|
||||||
this.getProductionChanges(),
|
this.getProductionChanges(),
|
||||||
this.clientMetricsStore.countPreviousDayHourlyMetricsBuckets(),
|
this.countPreviousDayHourlyMetricsBuckets(),
|
||||||
this.featureStrategiesReadModel.getMaxFeatureEnvironmentStrategies(),
|
this.featureStrategiesReadModel.getMaxFeatureEnvironmentStrategies(),
|
||||||
this.featureStrategiesReadModel.getMaxConstraintValues(),
|
this.featureStrategiesReadModel.getMaxConstraintValues(),
|
||||||
this.featureStrategiesReadModel.getMaxConstraintsPerStrategy(),
|
this.featureStrategiesReadModel.getMaxConstraintsPerStrategy(),
|
||||||
@ -315,17 +307,72 @@ export class InstanceStatsService {
|
|||||||
maxConstraints: maxConstraints?.count ?? 0,
|
maxConstraints: maxConstraints?.count ?? 0,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
async clientAppCounts(): Promise<Partial<{ [key in TimeRange]: number }>> {
|
|
||||||
|
groupCount(): Promise<number> {
|
||||||
|
return this.groupStore.count();
|
||||||
|
}
|
||||||
|
|
||||||
|
roleCount(): Promise<number> {
|
||||||
|
return this.roleStore.count();
|
||||||
|
}
|
||||||
|
|
||||||
|
customRolesCount(): Promise<number> {
|
||||||
|
return this.roleStore.filteredCount({ type: CUSTOM_ROOT_ROLE_TYPE });
|
||||||
|
}
|
||||||
|
|
||||||
|
customRolesCountInUse(): Promise<number> {
|
||||||
|
return this.roleStore.filteredCountInUse({
|
||||||
|
type: CUSTOM_ROOT_ROLE_TYPE,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
segmentCount(): Promise<number> {
|
||||||
|
return this.segmentStore.count();
|
||||||
|
}
|
||||||
|
|
||||||
|
contextFieldCount(): Promise<number> {
|
||||||
|
return this.contextFieldStore.count();
|
||||||
|
}
|
||||||
|
|
||||||
|
strategiesCount(): Promise<number> {
|
||||||
|
return this.strategyStore.count();
|
||||||
|
}
|
||||||
|
|
||||||
|
environmentCount(): Promise<number> {
|
||||||
|
return this.environmentStore.count();
|
||||||
|
}
|
||||||
|
|
||||||
|
countPreviousDayHourlyMetricsBuckets(): Promise<{
|
||||||
|
enabledCount: number;
|
||||||
|
variantCount: number;
|
||||||
|
}> {
|
||||||
|
return this.clientMetricsStore.countPreviousDayHourlyMetricsBuckets();
|
||||||
|
}
|
||||||
|
|
||||||
|
countApiTokensByType(): Promise<Map<string, number>> {
|
||||||
|
return this.apiTokenStore.countByType();
|
||||||
|
}
|
||||||
|
|
||||||
|
getRegisteredUsers(): Promise<number> {
|
||||||
|
return this.userStore.count();
|
||||||
|
}
|
||||||
|
|
||||||
|
countServiceAccounts(): Promise<number> {
|
||||||
|
return this.userStore.countServiceAccounts();
|
||||||
|
}
|
||||||
|
|
||||||
|
async getLabeledAppCounts(): Promise<
|
||||||
|
Partial<{ [key in TimeRange]: number }>
|
||||||
|
> {
|
||||||
|
const [t7d, t30d, allTime] = await Promise.all([
|
||||||
|
this.clientInstanceStore.getDistinctApplicationsCount(7),
|
||||||
|
this.clientInstanceStore.getDistinctApplicationsCount(30),
|
||||||
|
this.clientInstanceStore.getDistinctApplicationsCount(),
|
||||||
|
]);
|
||||||
this.appCount = {
|
this.appCount = {
|
||||||
'7d': await this.fromPrometheus('client_apps_total', {
|
'7d': t7d,
|
||||||
range: '7d',
|
'30d': t30d,
|
||||||
}),
|
allTime,
|
||||||
'30d': await this.fromPrometheus('client_apps_total', {
|
|
||||||
range: '30d',
|
|
||||||
}),
|
|
||||||
allTime: await this.fromPrometheus('client_apps_total', {
|
|
||||||
range: 'allTime',
|
|
||||||
}),
|
|
||||||
};
|
};
|
||||||
return this.appCount;
|
return this.appCount;
|
||||||
}
|
}
|
||||||
|
@ -15,7 +15,11 @@ import {
|
|||||||
FEATURE_UPDATED,
|
FEATURE_UPDATED,
|
||||||
PROJECT_ENVIRONMENT_REMOVED,
|
PROJECT_ENVIRONMENT_REMOVED,
|
||||||
} from './types/events';
|
} from './types/events';
|
||||||
import { createMetricsMonitor } from './metrics';
|
import {
|
||||||
|
createMetricsMonitor,
|
||||||
|
registerPrometheusMetrics,
|
||||||
|
registerPrometheusPostgresMetrics,
|
||||||
|
} from './metrics';
|
||||||
import createStores from '../test/fixtures/store';
|
import createStores from '../test/fixtures/store';
|
||||||
import { InstanceStatsService } from './features/instance-stats/instance-stats-service';
|
import { InstanceStatsService } from './features/instance-stats/instance-stats-service';
|
||||||
import VersionService from './services/version-service';
|
import VersionService from './services/version-service';
|
||||||
@ -46,6 +50,7 @@ let schedulerService: SchedulerService;
|
|||||||
let featureLifeCycleStore: IFeatureLifecycleStore;
|
let featureLifeCycleStore: IFeatureLifecycleStore;
|
||||||
let featureLifeCycleReadModel: IFeatureLifecycleReadModel;
|
let featureLifeCycleReadModel: IFeatureLifecycleReadModel;
|
||||||
let db: ITestDb;
|
let db: ITestDb;
|
||||||
|
let refreshDbMetrics: () => Promise<void>;
|
||||||
|
|
||||||
beforeAll(async () => {
|
beforeAll(async () => {
|
||||||
const config = createTestConfig({
|
const config = createTestConfig({
|
||||||
@ -102,16 +107,16 @@ beforeAll(async () => {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
await monitor.startMonitoring(
|
const { collectDbMetrics, collectStaticCounters } =
|
||||||
config,
|
registerPrometheusMetrics(
|
||||||
stores,
|
config,
|
||||||
'4.0.0',
|
stores,
|
||||||
eventBus,
|
'4.0.0',
|
||||||
statsService,
|
eventBus,
|
||||||
schedulerService,
|
statsService,
|
||||||
// @ts-ignore - We don't want a full knex implementation for our tests, it's enough that it actually yields the numbers we want.
|
);
|
||||||
metricsDbConf,
|
refreshDbMetrics = collectDbMetrics;
|
||||||
);
|
await collectStaticCounters();
|
||||||
});
|
});
|
||||||
|
|
||||||
afterAll(async () => {
|
afterAll(async () => {
|
||||||
@ -212,7 +217,7 @@ test('should collect metrics for function timings', async () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
test('should collect metrics for feature flag size', async () => {
|
test('should collect metrics for feature flag size', async () => {
|
||||||
await statsService.dbMetrics.refreshDbMetrics();
|
await refreshDbMetrics();
|
||||||
const metrics = await prometheusRegister.metrics();
|
const metrics = await prometheusRegister.metrics();
|
||||||
expect(metrics).toMatch(/feature_toggles_total\{version="(.*)"\} 0/);
|
expect(metrics).toMatch(/feature_toggles_total\{version="(.*)"\} 0/);
|
||||||
});
|
});
|
||||||
@ -223,12 +228,13 @@ test('should collect metrics for archived feature flag size', async () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
test('should collect metrics for total client apps', async () => {
|
test('should collect metrics for total client apps', async () => {
|
||||||
await statsService.dbMetrics.refreshDbMetrics();
|
await refreshDbMetrics();
|
||||||
const metrics = await prometheusRegister.metrics();
|
const metrics = await prometheusRegister.metrics();
|
||||||
expect(metrics).toMatch(/client_apps_total\{range="(.*)"\} 0/);
|
expect(metrics).toMatch(/client_apps_total\{range="(.*)"\} 0/);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('Should collect metrics for database', async () => {
|
test('Should collect metrics for database', async () => {
|
||||||
|
registerPrometheusPostgresMetrics(db.rawDatabase, eventBus, '15.0.0');
|
||||||
const metrics = await prometheusRegister.metrics();
|
const metrics = await prometheusRegister.metrics();
|
||||||
expect(metrics).toMatch(/db_pool_max/);
|
expect(metrics).toMatch(/db_pool_max/);
|
||||||
expect(metrics).toMatch(/db_pool_min/);
|
expect(metrics).toMatch(/db_pool_min/);
|
||||||
|
@ -25,7 +25,7 @@ import {
|
|||||||
PROJECT_DELETED,
|
PROJECT_DELETED,
|
||||||
} from './types/events';
|
} from './types/events';
|
||||||
import type { IUnleashConfig } from './types/option';
|
import type { IUnleashConfig } from './types/option';
|
||||||
import type { ISettingStore, IUnleashStores } from './types/stores';
|
import type { IUnleashStores } from './types/stores';
|
||||||
import { hoursToMilliseconds, minutesToMilliseconds } from 'date-fns';
|
import { hoursToMilliseconds, minutesToMilliseconds } from 'date-fns';
|
||||||
import type { InstanceStatsService } from './features/instance-stats/instance-stats-service';
|
import type { InstanceStatsService } from './features/instance-stats/instance-stats-service';
|
||||||
import type { IEnvironment, ISdkHeartbeat } from './types';
|
import type { IEnvironment, ISdkHeartbeat } from './types';
|
||||||
@ -37,6 +37,56 @@ import {
|
|||||||
} from './util/metrics';
|
} from './util/metrics';
|
||||||
import type { SchedulerService } from './services';
|
import type { SchedulerService } from './services';
|
||||||
import type { IClientMetricsEnv } from './features/metrics/client-metrics/client-metrics-store-v2-type';
|
import type { IClientMetricsEnv } from './features/metrics/client-metrics/client-metrics-store-v2-type';
|
||||||
|
import { DbMetricsMonitor } from './metrics-gauge';
|
||||||
|
|
||||||
|
export function registerPrometheusPostgresMetrics(
|
||||||
|
db: Knex,
|
||||||
|
eventBus: EventEmitter,
|
||||||
|
postgresVersion: string,
|
||||||
|
) {
|
||||||
|
if (db?.client) {
|
||||||
|
const dbPoolMin = createGauge({
|
||||||
|
name: 'db_pool_min',
|
||||||
|
help: 'Minimum DB pool size',
|
||||||
|
});
|
||||||
|
dbPoolMin.set(db.client.pool.min);
|
||||||
|
const dbPoolMax = createGauge({
|
||||||
|
name: 'db_pool_max',
|
||||||
|
help: 'Maximum DB pool size',
|
||||||
|
});
|
||||||
|
dbPoolMax.set(db.client.pool.max);
|
||||||
|
const dbPoolFree = createGauge({
|
||||||
|
name: 'db_pool_free',
|
||||||
|
help: 'Current free connections in DB pool',
|
||||||
|
});
|
||||||
|
const dbPoolUsed = createGauge({
|
||||||
|
name: 'db_pool_used',
|
||||||
|
help: 'Current connections in use in DB pool',
|
||||||
|
});
|
||||||
|
const dbPoolPendingCreates = createGauge({
|
||||||
|
name: 'db_pool_pending_creates',
|
||||||
|
help: 'how many asynchronous create calls are running in DB pool',
|
||||||
|
});
|
||||||
|
const dbPoolPendingAcquires = createGauge({
|
||||||
|
name: 'db_pool_pending_acquires',
|
||||||
|
help: 'how many acquires are waiting for a resource to be released in DB pool',
|
||||||
|
});
|
||||||
|
|
||||||
|
eventBus.on(DB_POOL_UPDATE, (data) => {
|
||||||
|
dbPoolFree.set(data.free);
|
||||||
|
dbPoolUsed.set(data.used);
|
||||||
|
dbPoolPendingCreates.set(data.pendingCreates);
|
||||||
|
dbPoolPendingAcquires.set(data.pendingAcquires);
|
||||||
|
});
|
||||||
|
|
||||||
|
const database_version = createGauge({
|
||||||
|
name: 'postgres_version',
|
||||||
|
help: 'Which version of postgres is running (SHOW server_version)',
|
||||||
|
labelNames: ['version'],
|
||||||
|
});
|
||||||
|
database_version.labels({ version: postgresVersion }).set(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export function registerPrometheusMetrics(
|
export function registerPrometheusMetrics(
|
||||||
config: IUnleashConfig,
|
config: IUnleashConfig,
|
||||||
@ -60,8 +110,8 @@ export function registerPrometheusMetrics(
|
|||||||
};
|
};
|
||||||
|
|
||||||
const { eventStore, environmentStore } = stores;
|
const { eventStore, environmentStore } = stores;
|
||||||
const { flagResolver } = config;
|
const { flagResolver, db } = config;
|
||||||
const dbMetrics = instanceStatsService.dbMetrics;
|
const dbMetrics = new DbMetricsMonitor(config);
|
||||||
|
|
||||||
const cachedEnvironments: () => Promise<IEnvironment[]> = memoizee(
|
const cachedEnvironments: () => Promise<IEnvironment[]> = memoizee(
|
||||||
async () => environmentStore.getAll(),
|
async () => environmentStore.getAll(),
|
||||||
@ -124,10 +174,7 @@ export function registerPrometheusMetrics(
|
|||||||
name: 'feature_toggles_total',
|
name: 'feature_toggles_total',
|
||||||
help: 'Number of feature flags',
|
help: 'Number of feature flags',
|
||||||
labelNames: ['version'],
|
labelNames: ['version'],
|
||||||
query: () =>
|
query: () => instanceStatsService.getToggleCount(),
|
||||||
stores.featureToggleStore.count({
|
|
||||||
archived: false,
|
|
||||||
}),
|
|
||||||
map: (value) => ({ value, labels: { version } }),
|
map: (value) => ({ value, labels: { version } }),
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -268,18 +315,7 @@ export function registerPrometheusMetrics(
|
|||||||
name: 'client_apps_total',
|
name: 'client_apps_total',
|
||||||
help: 'Number of registered client apps aggregated by range by last seen',
|
help: 'Number of registered client apps aggregated by range by last seen',
|
||||||
labelNames: ['range'],
|
labelNames: ['range'],
|
||||||
query: async () => {
|
query: () => instanceStatsService.getLabeledAppCounts(),
|
||||||
const [t7d, t30d, allTime] = await Promise.all([
|
|
||||||
stores.clientInstanceStore.getDistinctApplicationsCount(7),
|
|
||||||
stores.clientInstanceStore.getDistinctApplicationsCount(30),
|
|
||||||
stores.clientInstanceStore.getDistinctApplicationsCount(),
|
|
||||||
]);
|
|
||||||
return {
|
|
||||||
'7d': t7d,
|
|
||||||
'30d': t30d,
|
|
||||||
allTime,
|
|
||||||
};
|
|
||||||
},
|
|
||||||
map: (result) =>
|
map: (result) =>
|
||||||
Object.entries(result).map(([range, count]) => ({
|
Object.entries(result).map(([range, count]) => ({
|
||||||
value: count,
|
value: count,
|
||||||
@ -735,13 +771,10 @@ export function registerPrometheusMetrics(
|
|||||||
addonEventsHandledCounter.increment({ result, destination });
|
addonEventsHandledCounter.increment({ result, destination });
|
||||||
});
|
});
|
||||||
|
|
||||||
// return an update function (temporarily) to allow for manual refresh
|
|
||||||
return {
|
return {
|
||||||
|
collectDbMetrics: dbMetrics.refreshDbMetrics,
|
||||||
collectStaticCounters: async () => {
|
collectStaticCounters: async () => {
|
||||||
try {
|
try {
|
||||||
dbMetrics.refreshDbMetrics();
|
|
||||||
|
|
||||||
const stats = await instanceStatsService.getStats();
|
|
||||||
const [
|
const [
|
||||||
maxConstraintValuesResult,
|
maxConstraintValuesResult,
|
||||||
maxConstraintsPerStrategyResult,
|
maxConstraintsPerStrategyResult,
|
||||||
@ -773,13 +806,17 @@ export function registerPrometheusMetrics(
|
|||||||
]);
|
]);
|
||||||
|
|
||||||
featureTogglesArchivedTotal.reset();
|
featureTogglesArchivedTotal.reset();
|
||||||
featureTogglesArchivedTotal.set(stats.archivedFeatureToggles);
|
featureTogglesArchivedTotal.set(
|
||||||
|
await instanceStatsService.getArchivedToggleCount(),
|
||||||
|
);
|
||||||
|
|
||||||
usersTotal.reset();
|
usersTotal.reset();
|
||||||
usersTotal.set(stats.users);
|
usersTotal.set(await instanceStatsService.getRegisteredUsers());
|
||||||
|
|
||||||
serviceAccounts.reset();
|
serviceAccounts.reset();
|
||||||
serviceAccounts.set(stats.serviceAccounts);
|
serviceAccounts.set(
|
||||||
|
await instanceStatsService.countServiceAccounts(),
|
||||||
|
);
|
||||||
|
|
||||||
stageDurationByProject.forEach((stage) => {
|
stageDurationByProject.forEach((stage) => {
|
||||||
featureLifecycleStageDuration
|
featureLifecycleStageDuration
|
||||||
@ -802,7 +839,10 @@ export function registerPrometheusMetrics(
|
|||||||
|
|
||||||
apiTokens.reset();
|
apiTokens.reset();
|
||||||
|
|
||||||
for (const [type, value] of stats.apiTokens) {
|
for (const [
|
||||||
|
type,
|
||||||
|
value,
|
||||||
|
] of await instanceStatsService.countApiTokensByType()) {
|
||||||
apiTokens.labels({ type }).set(value);
|
apiTokens.labels({ type }).set(value);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -887,67 +927,84 @@ export function registerPrometheusMetrics(
|
|||||||
resourceLimit.labels({ resource }).set(limit);
|
resourceLimit.labels({ resource }).set(limit);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const previousDayMetricsBucketsCount =
|
||||||
|
await instanceStatsService.countPreviousDayHourlyMetricsBuckets();
|
||||||
enabledMetricsBucketsPreviousDay.reset();
|
enabledMetricsBucketsPreviousDay.reset();
|
||||||
enabledMetricsBucketsPreviousDay.set(
|
enabledMetricsBucketsPreviousDay.set(
|
||||||
stats.previousDayMetricsBucketsCount.enabledCount,
|
previousDayMetricsBucketsCount.enabledCount,
|
||||||
);
|
);
|
||||||
variantMetricsBucketsPreviousDay.reset();
|
variantMetricsBucketsPreviousDay.reset();
|
||||||
variantMetricsBucketsPreviousDay.set(
|
variantMetricsBucketsPreviousDay.set(
|
||||||
stats.previousDayMetricsBucketsCount.variantCount,
|
previousDayMetricsBucketsCount.variantCount,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const activeUsers = await instanceStatsService.getActiveUsers();
|
||||||
usersActive7days.reset();
|
usersActive7days.reset();
|
||||||
usersActive7days.set(stats.activeUsers.last7);
|
usersActive7days.set(activeUsers.last7);
|
||||||
usersActive30days.reset();
|
usersActive30days.reset();
|
||||||
usersActive30days.set(stats.activeUsers.last30);
|
usersActive30days.set(activeUsers.last30);
|
||||||
usersActive60days.reset();
|
usersActive60days.reset();
|
||||||
usersActive60days.set(stats.activeUsers.last60);
|
usersActive60days.set(activeUsers.last60);
|
||||||
usersActive90days.reset();
|
usersActive90days.reset();
|
||||||
usersActive90days.set(stats.activeUsers.last90);
|
usersActive90days.set(activeUsers.last90);
|
||||||
|
|
||||||
|
const productionChanges =
|
||||||
|
await instanceStatsService.getProductionChanges();
|
||||||
productionChanges30.reset();
|
productionChanges30.reset();
|
||||||
productionChanges30.set(stats.productionChanges.last30);
|
productionChanges30.set(productionChanges.last30);
|
||||||
productionChanges60.reset();
|
productionChanges60.reset();
|
||||||
productionChanges60.set(stats.productionChanges.last60);
|
productionChanges60.set(productionChanges.last60);
|
||||||
productionChanges90.reset();
|
productionChanges90.reset();
|
||||||
productionChanges90.set(stats.productionChanges.last90);
|
productionChanges90.set(productionChanges.last90);
|
||||||
|
|
||||||
|
const projects =
|
||||||
|
await instanceStatsService.getProjectModeCount();
|
||||||
projectsTotal.reset();
|
projectsTotal.reset();
|
||||||
stats.projects.forEach((projectStat) => {
|
projects.forEach((projectStat) => {
|
||||||
projectsTotal
|
projectsTotal
|
||||||
.labels({ mode: projectStat.mode })
|
.labels({ mode: projectStat.mode })
|
||||||
.set(projectStat.count);
|
.set(projectStat.count);
|
||||||
});
|
});
|
||||||
|
|
||||||
environmentsTotal.reset();
|
environmentsTotal.reset();
|
||||||
environmentsTotal.set(stats.environments);
|
environmentsTotal.set(
|
||||||
|
await instanceStatsService.environmentCount(),
|
||||||
|
);
|
||||||
|
|
||||||
groupsTotal.reset();
|
groupsTotal.reset();
|
||||||
groupsTotal.set(stats.groups);
|
groupsTotal.set(await instanceStatsService.groupCount());
|
||||||
|
|
||||||
rolesTotal.reset();
|
rolesTotal.reset();
|
||||||
rolesTotal.set(stats.roles);
|
rolesTotal.set(await instanceStatsService.roleCount());
|
||||||
|
|
||||||
customRootRolesTotal.reset();
|
customRootRolesTotal.reset();
|
||||||
customRootRolesTotal.set(stats.customRootRoles);
|
customRootRolesTotal.set(
|
||||||
|
await instanceStatsService.customRolesCount(),
|
||||||
|
);
|
||||||
|
|
||||||
customRootRolesInUseTotal.reset();
|
customRootRolesInUseTotal.reset();
|
||||||
customRootRolesInUseTotal.set(stats.customRootRolesInUse);
|
customRootRolesInUseTotal.set(
|
||||||
|
await instanceStatsService.customRolesCountInUse(),
|
||||||
|
);
|
||||||
|
|
||||||
segmentsTotal.reset();
|
segmentsTotal.reset();
|
||||||
segmentsTotal.set(stats.segments);
|
segmentsTotal.set(await instanceStatsService.segmentCount());
|
||||||
|
|
||||||
contextTotal.reset();
|
contextTotal.reset();
|
||||||
contextTotal.set(stats.contextFields);
|
contextTotal.set(
|
||||||
|
await instanceStatsService.contextFieldCount(),
|
||||||
|
);
|
||||||
|
|
||||||
strategiesTotal.reset();
|
strategiesTotal.reset();
|
||||||
strategiesTotal.set(stats.strategies);
|
strategiesTotal.set(
|
||||||
|
await instanceStatsService.strategiesCount(),
|
||||||
|
);
|
||||||
|
|
||||||
samlEnabled.reset();
|
samlEnabled.reset();
|
||||||
samlEnabled.set(stats.SAMLenabled ? 1 : 0);
|
samlEnabled.set((await instanceStatsService.hasSAML()) ? 1 : 0);
|
||||||
|
|
||||||
oidcEnabled.reset();
|
oidcEnabled.reset();
|
||||||
oidcEnabled.set(stats.OIDCenabled ? 1 : 0);
|
oidcEnabled.set((await instanceStatsService.hasOIDC()) ? 1 : 0);
|
||||||
|
|
||||||
rateLimits.reset();
|
rateLimits.reset();
|
||||||
rateLimits
|
rateLimits
|
||||||
@ -1026,22 +1083,21 @@ export default class MetricsMonitor {
|
|||||||
|
|
||||||
collectDefaultMetrics();
|
collectDefaultMetrics();
|
||||||
|
|
||||||
const { collectStaticCounters } = registerPrometheusMetrics(
|
const { collectStaticCounters, collectDbMetrics } =
|
||||||
config,
|
registerPrometheusMetrics(
|
||||||
stores,
|
config,
|
||||||
version,
|
stores,
|
||||||
eventBus,
|
version,
|
||||||
instanceStatsService,
|
eventBus,
|
||||||
);
|
instanceStatsService,
|
||||||
|
);
|
||||||
|
|
||||||
await this.registerPrometheusDbMetrics(
|
const postgresVersion = await stores.settingStore.postgresVersion();
|
||||||
db,
|
registerPrometheusPostgresMetrics(db, eventBus, postgresVersion);
|
||||||
eventBus,
|
|
||||||
stores.settingStore,
|
|
||||||
);
|
|
||||||
|
|
||||||
await schedulerService.schedule(
|
await schedulerService.schedule(
|
||||||
collectStaticCounters.bind(this),
|
async () =>
|
||||||
|
Promise.all([collectStaticCounters(), collectDbMetrics()]),
|
||||||
hoursToMilliseconds(2),
|
hoursToMilliseconds(2),
|
||||||
'collectStaticCounters',
|
'collectStaticCounters',
|
||||||
);
|
);
|
||||||
@ -1056,56 +1112,6 @@ export default class MetricsMonitor {
|
|||||||
return Promise.resolve();
|
return Promise.resolve();
|
||||||
}
|
}
|
||||||
|
|
||||||
async registerPrometheusDbMetrics(
|
|
||||||
db: Knex,
|
|
||||||
eventBus: EventEmitter,
|
|
||||||
settingStore: ISettingStore,
|
|
||||||
): Promise<void> {
|
|
||||||
if (db?.client) {
|
|
||||||
const dbPoolMin = createGauge({
|
|
||||||
name: 'db_pool_min',
|
|
||||||
help: 'Minimum DB pool size',
|
|
||||||
});
|
|
||||||
dbPoolMin.set(db.client.pool.min);
|
|
||||||
const dbPoolMax = createGauge({
|
|
||||||
name: 'db_pool_max',
|
|
||||||
help: 'Maximum DB pool size',
|
|
||||||
});
|
|
||||||
dbPoolMax.set(db.client.pool.max);
|
|
||||||
const dbPoolFree = createGauge({
|
|
||||||
name: 'db_pool_free',
|
|
||||||
help: 'Current free connections in DB pool',
|
|
||||||
});
|
|
||||||
const dbPoolUsed = createGauge({
|
|
||||||
name: 'db_pool_used',
|
|
||||||
help: 'Current connections in use in DB pool',
|
|
||||||
});
|
|
||||||
const dbPoolPendingCreates = createGauge({
|
|
||||||
name: 'db_pool_pending_creates',
|
|
||||||
help: 'how many asynchronous create calls are running in DB pool',
|
|
||||||
});
|
|
||||||
const dbPoolPendingAcquires = createGauge({
|
|
||||||
name: 'db_pool_pending_acquires',
|
|
||||||
help: 'how many acquires are waiting for a resource to be released in DB pool',
|
|
||||||
});
|
|
||||||
|
|
||||||
eventBus.on(DB_POOL_UPDATE, (data) => {
|
|
||||||
dbPoolFree.set(data.free);
|
|
||||||
dbPoolUsed.set(data.used);
|
|
||||||
dbPoolPendingCreates.set(data.pendingCreates);
|
|
||||||
dbPoolPendingAcquires.set(data.pendingAcquires);
|
|
||||||
});
|
|
||||||
|
|
||||||
const postgresVersion = await settingStore.postgresVersion();
|
|
||||||
const database_version = createGauge({
|
|
||||||
name: 'postgres_version',
|
|
||||||
help: 'Which version of postgres is running (SHOW server_version)',
|
|
||||||
labelNames: ['version'],
|
|
||||||
});
|
|
||||||
database_version.labels({ version: postgresVersion }).set(1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
|
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
|
||||||
registerPoolMetrics(pool: any, eventBus: EventEmitter) {
|
registerPoolMetrics(pool: any, eventBus: EventEmitter) {
|
||||||
try {
|
try {
|
||||||
|
@ -11,6 +11,7 @@ import { registerPrometheusMetrics } from '../../../../lib/metrics';
|
|||||||
let app: IUnleashTest;
|
let app: IUnleashTest;
|
||||||
let db: ITestDb;
|
let db: ITestDb;
|
||||||
let stores: IUnleashStores;
|
let stores: IUnleashStores;
|
||||||
|
let refreshDbMetrics: () => Promise<void>;
|
||||||
|
|
||||||
beforeAll(async () => {
|
beforeAll(async () => {
|
||||||
db = await dbInit('instance_admin_api_serial', getLogger);
|
db = await dbInit('instance_admin_api_serial', getLogger);
|
||||||
@ -28,13 +29,14 @@ beforeAll(async () => {
|
|||||||
db.rawDatabase,
|
db.rawDatabase,
|
||||||
);
|
);
|
||||||
|
|
||||||
registerPrometheusMetrics(
|
const { collectDbMetrics } = registerPrometheusMetrics(
|
||||||
app.config,
|
app.config,
|
||||||
stores,
|
stores,
|
||||||
undefined as unknown as string,
|
undefined as unknown as string,
|
||||||
app.config.eventBus,
|
app.config.eventBus,
|
||||||
app.services.instanceStatsService,
|
app.services.instanceStatsService,
|
||||||
);
|
);
|
||||||
|
refreshDbMetrics = collectDbMetrics;
|
||||||
});
|
});
|
||||||
|
|
||||||
afterAll(async () => {
|
afterAll(async () => {
|
||||||
@ -48,7 +50,7 @@ test('should return instance statistics', async () => {
|
|||||||
createdByUserId: 9999,
|
createdByUserId: 9999,
|
||||||
});
|
});
|
||||||
|
|
||||||
await app.services.instanceStatsService.dbMetrics.refreshDbMetrics();
|
await refreshDbMetrics();
|
||||||
|
|
||||||
return app.request
|
return app.request
|
||||||
.get('/api/admin/instance-admin/statistics')
|
.get('/api/admin/instance-admin/statistics')
|
||||||
|
Loading…
Reference in New Issue
Block a user