From b55d677d1e4dd5e653fae7c23a8f2bd131231c67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nuno=20G=C3=B3is?= Date: Tue, 8 Aug 2023 08:14:40 +0100 Subject: [PATCH] feat: add prom metric for total custom root roles in use (#4438) https://linear.app/unleash/issue/2-1311/add-a-new-prometheus-metric-with-custom-root-roles-in-use As a follow-up to https://github.com/Unleash/unleash/pull/4435, this PR adds a metric for total custom root roles in use by at least one entity: users, service accounts, groups. `custom_root_roles_in_use_total` Output from `http://localhost:4242/internal-backstage/prometheus`: ``` # HELP process_cpu_user_seconds_total Total user CPU time spent in seconds. # TYPE process_cpu_user_seconds_total counter process_cpu_user_seconds_total 0.060755 # HELP process_cpu_system_seconds_total Total system CPU time spent in seconds. # TYPE process_cpu_system_seconds_total counter process_cpu_system_seconds_total 0.01666 # HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. # TYPE process_cpu_seconds_total counter process_cpu_seconds_total 0.077415 # HELP process_start_time_seconds Start time of the process since unix epoch in seconds. # TYPE process_start_time_seconds gauge process_start_time_seconds 1691420275 # HELP process_resident_memory_bytes Resident memory size in bytes. # TYPE process_resident_memory_bytes gauge process_resident_memory_bytes 199196672 # HELP nodejs_eventloop_lag_seconds Lag of event loop in seconds. # TYPE nodejs_eventloop_lag_seconds gauge nodejs_eventloop_lag_seconds 0 # HELP nodejs_eventloop_lag_min_seconds The minimum recorded event loop delay. # TYPE nodejs_eventloop_lag_min_seconds gauge nodejs_eventloop_lag_min_seconds 0.009076736 # HELP nodejs_eventloop_lag_max_seconds The maximum recorded event loop delay. # TYPE nodejs_eventloop_lag_max_seconds gauge nodejs_eventloop_lag_max_seconds 0.037683199 # HELP nodejs_eventloop_lag_mean_seconds The mean of the recorded event loop delays. # TYPE nodejs_eventloop_lag_mean_seconds gauge nodejs_eventloop_lag_mean_seconds 0.011063251638989169 # HELP nodejs_eventloop_lag_stddev_seconds The standard deviation of the recorded event loop delays. # TYPE nodejs_eventloop_lag_stddev_seconds gauge nodejs_eventloop_lag_stddev_seconds 0.0013618102764025837 # HELP nodejs_eventloop_lag_p50_seconds The 50th percentile of the recorded event loop delays. # TYPE nodejs_eventloop_lag_p50_seconds gauge nodejs_eventloop_lag_p50_seconds 0.011051007 # HELP nodejs_eventloop_lag_p90_seconds The 90th percentile of the recorded event loop delays. # TYPE nodejs_eventloop_lag_p90_seconds gauge nodejs_eventloop_lag_p90_seconds 0.011321343 # HELP nodejs_eventloop_lag_p99_seconds The 99th percentile of the recorded event loop delays. # TYPE nodejs_eventloop_lag_p99_seconds gauge nodejs_eventloop_lag_p99_seconds 0.013688831 # HELP nodejs_active_resources Number of active resources that are currently keeping the event loop alive, grouped by async resource type. # TYPE nodejs_active_resources gauge nodejs_active_resources{type="FSReqCallback"} 1 nodejs_active_resources{type="TTYWrap"} 3 nodejs_active_resources{type="TCPSocketWrap"} 5 nodejs_active_resources{type="TCPServerWrap"} 1 nodejs_active_resources{type="Timeout"} 1 nodejs_active_resources{type="Immediate"} 1 # HELP nodejs_active_resources_total Total number of active resources. # TYPE nodejs_active_resources_total gauge nodejs_active_resources_total 12 # HELP nodejs_active_handles Number of active libuv handles grouped by handle type. Every handle type is C++ class name. # TYPE nodejs_active_handles gauge nodejs_active_handles{type="WriteStream"} 2 nodejs_active_handles{type="ReadStream"} 1 nodejs_active_handles{type="Socket"} 5 nodejs_active_handles{type="Server"} 1 # HELP nodejs_active_handles_total Total number of active handles. # TYPE nodejs_active_handles_total gauge nodejs_active_handles_total 9 # HELP nodejs_active_requests Number of active libuv requests grouped by request type. Every request type is C++ class name. # TYPE nodejs_active_requests gauge nodejs_active_requests{type="FSReqCallback"} 1 # HELP nodejs_active_requests_total Total number of active requests. # TYPE nodejs_active_requests_total gauge nodejs_active_requests_total 1 # HELP nodejs_heap_size_total_bytes Process heap size from Node.js in bytes. # TYPE nodejs_heap_size_total_bytes gauge nodejs_heap_size_total_bytes 118587392 # HELP nodejs_heap_size_used_bytes Process heap size used from Node.js in bytes. # TYPE nodejs_heap_size_used_bytes gauge nodejs_heap_size_used_bytes 89642552 # HELP nodejs_external_memory_bytes Node.js external memory size in bytes. # TYPE nodejs_external_memory_bytes gauge nodejs_external_memory_bytes 1601594 # HELP nodejs_heap_space_size_total_bytes Process heap space size total from Node.js in bytes. # TYPE nodejs_heap_space_size_total_bytes gauge nodejs_heap_space_size_total_bytes{space="read_only"} 0 nodejs_heap_space_size_total_bytes{space="old"} 70139904 nodejs_heap_space_size_total_bytes{space="code"} 3588096 nodejs_heap_space_size_total_bytes{space="map"} 2899968 nodejs_heap_space_size_total_bytes{space="large_object"} 7258112 nodejs_heap_space_size_total_bytes{space="code_large_object"} 1146880 nodejs_heap_space_size_total_bytes{space="new_large_object"} 0 nodejs_heap_space_size_total_bytes{space="new"} 33554432 # HELP nodejs_heap_space_size_used_bytes Process heap space size used from Node.js in bytes. # TYPE nodejs_heap_space_size_used_bytes gauge nodejs_heap_space_size_used_bytes{space="read_only"} 0 nodejs_heap_space_size_used_bytes{space="old"} 66992120 nodejs_heap_space_size_used_bytes{space="code"} 2892640 nodejs_heap_space_size_used_bytes{space="map"} 2519280 nodejs_heap_space_size_used_bytes{space="large_object"} 7026824 nodejs_heap_space_size_used_bytes{space="code_large_object"} 983200 nodejs_heap_space_size_used_bytes{space="new_large_object"} 0 nodejs_heap_space_size_used_bytes{space="new"} 9236136 # HELP nodejs_heap_space_size_available_bytes Process heap space size available from Node.js in bytes. # TYPE nodejs_heap_space_size_available_bytes gauge nodejs_heap_space_size_available_bytes{space="read_only"} 0 nodejs_heap_space_size_available_bytes{space="old"} 1898360 nodejs_heap_space_size_available_bytes{space="code"} 7328 nodejs_heap_space_size_available_bytes{space="map"} 327888 nodejs_heap_space_size_available_bytes{space="large_object"} 0 nodejs_heap_space_size_available_bytes{space="code_large_object"} 0 nodejs_heap_space_size_available_bytes{space="new_large_object"} 16495616 nodejs_heap_space_size_available_bytes{space="new"} 7259480 # HELP nodejs_version_info Node.js version info. # TYPE nodejs_version_info gauge nodejs_version_info{version="v18.16.0",major="18",minor="16",patch="0"} 1 # HELP nodejs_gc_duration_seconds Garbage collection duration by kind, one of major, minor, incremental or weakcb. # TYPE nodejs_gc_duration_seconds histogram # HELP http_request_duration_milliseconds App response time # TYPE http_request_duration_milliseconds summary # HELP db_query_duration_seconds DB query duration time # TYPE db_query_duration_seconds summary db_query_duration_seconds{quantile="0.1",store="api-tokens",action="getAllActive"} 0.03091475 db_query_duration_seconds{quantile="0.5",store="api-tokens",action="getAllActive"} 0.03091475 db_query_duration_seconds{quantile="0.9",store="api-tokens",action="getAllActive"} 0.03091475 db_query_duration_seconds{quantile="0.95",store="api-tokens",action="getAllActive"} 0.03091475 db_query_duration_seconds{quantile="0.99",store="api-tokens",action="getAllActive"} 0.03091475 db_query_duration_seconds_sum{store="api-tokens",action="getAllActive"} 0.03091475 db_query_duration_seconds_count{store="api-tokens",action="getAllActive"} 1 # HELP feature_toggle_update_total Number of times a toggle has been updated. Environment label would be "n/a" when it is not available, e.g. when a feature toggle is created. # TYPE feature_toggle_update_total counter # HELP feature_toggle_usage_total Number of times a feature toggle has been used # TYPE feature_toggle_usage_total counter # HELP feature_toggles_total Number of feature toggles # TYPE feature_toggles_total gauge feature_toggles_total{version="5.3.0"} 31 # HELP users_total Number of users # TYPE users_total gauge users_total 1011 # HELP projects_total Number of projects # TYPE projects_total gauge projects_total 4 # HELP environments_total Number of environments # TYPE environments_total gauge environments_total 10 # HELP groups_total Number of groups # TYPE groups_total gauge groups_total 5 # HELP roles_total Number of roles # TYPE roles_total gauge roles_total 11 # HELP custom_root_roles_total Number of custom root roles # TYPE custom_root_roles_total gauge custom_root_roles_total 3 # HELP custom_root_roles_in_use_total Number of custom root roles in use # TYPE custom_root_roles_in_use_total gauge custom_root_roles_in_use_total 2 # HELP segments_total Number of segments # TYPE segments_total gauge segments_total 5 # HELP context_total Number of context # TYPE context_total gauge context_total 7 # HELP strategies_total Number of strategies # TYPE strategies_total gauge strategies_total 5 # HELP client_apps_total Number of registered client apps aggregated by range by last seen # TYPE client_apps_total gauge client_apps_total{range="allTime"} 0 client_apps_total{range="30d"} 0 client_apps_total{range="7d"} 0 # HELP saml_enabled Whether SAML is enabled # TYPE saml_enabled gauge saml_enabled 1 # HELP oidc_enabled Whether OIDC is enabled # TYPE oidc_enabled gauge oidc_enabled 0 # HELP client_sdk_versions Which sdk versions are being used # TYPE client_sdk_versions counter # HELP optimal_304_diffing Count the Optimal 304 diffing with status # TYPE optimal_304_diffing counter # HELP db_pool_min Minimum DB pool size # TYPE db_pool_min gauge db_pool_min 0 # HELP db_pool_max Maximum DB pool size # TYPE db_pool_max gauge db_pool_max 4 # HELP db_pool_free Current free connections in DB pool # TYPE db_pool_free gauge db_pool_free 0 # HELP db_pool_used Current connections in use in DB pool # TYPE db_pool_used gauge db_pool_used 4 # HELP db_pool_pending_creates how many asynchronous create calls are running in DB pool # TYPE db_pool_pending_creates gauge db_pool_pending_creates 0 # HELP db_pool_pending_acquires how many acquires are waiting for a resource to be released in DB pool # TYPE db_pool_pending_acquires gauge db_pool_pending_acquires 24 ``` --- .../admin/instance-privacy/InstancePrivacy.tsx | 2 ++ src/lib/db/role-store.ts | 13 +++++++++++++ src/lib/metrics.ts | 8 ++++++++ src/lib/routes/admin-api/instance-admin.ts | 3 ++- src/lib/services/instance-stats-service.ts | 4 ++++ src/lib/services/version-service.ts | 3 +++ src/lib/types/stores/role-store.ts | 1 + src/test/fixtures/fake-role-store.ts | 4 ++++ 8 files changed, 37 insertions(+), 1 deletion(-) diff --git a/frontend/src/component/admin/instance-privacy/InstancePrivacy.tsx b/frontend/src/component/admin/instance-privacy/InstancePrivacy.tsx index e2c08f240d..cc78850d0d 100644 --- a/frontend/src/component/admin/instance-privacy/InstancePrivacy.tsx +++ b/frontend/src/component/admin/instance-privacy/InstancePrivacy.tsx @@ -44,6 +44,8 @@ const featureCollectionDetails = { Roles: 'The number of custom roles defined in your instance', 'Custom Root Roles': 'The number of custom root roles defined in your instance', + 'Custom Root Roles In Use': + 'The number of custom root roles that are in use by entities (users, groups, service accounts)', Environments: 'The number of environments in your instance', Segments: 'The number of segments defined in your instance', Strategies: 'The number of strategies defined in your instance', diff --git a/src/lib/db/role-store.ts b/src/lib/db/role-store.ts index 104d72329d..f5386b4e68 100644 --- a/src/lib/db/role-store.ts +++ b/src/lib/db/role-store.ts @@ -64,6 +64,19 @@ export default class RoleStore implements IRoleStore { .then((res) => Number(res[0].count)); } + async filteredCountInUse(filter: Partial): Promise { + return this.db + .from(T.ROLES) + .countDistinct('roles.id') + .leftJoin('role_user as ru', 'roles.id', 'ru.role_id') + .leftJoin('groups as g', 'roles.id', 'g.root_role_id') + .where(filter) + .andWhere((qb) => + qb.whereNotNull('ru.role_id').orWhereNotNull('g.root_role_id'), + ) + .then((res) => Number(res[0].count)); + } + async create(role: ICustomRoleInsert): Promise { const row = await this.db(T.ROLES) .insert({ diff --git a/src/lib/metrics.ts b/src/lib/metrics.ts index 6249e6a848..ee8ee790e9 100644 --- a/src/lib/metrics.ts +++ b/src/lib/metrics.ts @@ -109,6 +109,11 @@ export default class MetricsMonitor { help: 'Number of custom root roles', }); + const customRootRolesInUseTotal = new client.Gauge({ + name: 'custom_root_roles_in_use_total', + help: 'Number of custom root roles in use', + }); + const segmentsTotal = new client.Gauge({ name: 'segments_total', help: 'Number of segments', @@ -177,6 +182,9 @@ export default class MetricsMonitor { customRootRolesTotal.reset(); customRootRolesTotal.set(stats.customRootRoles); + customRootRolesInUseTotal.reset(); + customRootRolesInUseTotal.set(stats.customRootRolesInUse); + segmentsTotal.reset(); segmentsTotal.set(stats.segments); diff --git a/src/lib/routes/admin-api/instance-admin.ts b/src/lib/routes/admin-api/instance-admin.ts index 32942069bf..94c17bb9a8 100644 --- a/src/lib/routes/admin-api/instance-admin.ts +++ b/src/lib/routes/admin-api/instance-admin.ts @@ -99,7 +99,8 @@ class InstanceAdminController extends Controller { instanceId: 'ed3861ae-78f9-4e8c-8e57-b57efc15f82b', projects: 1, roles: 5, - customRootRoles: 1, + customRootRoles: 2, + customRootRolesInUse: 1, segments: 2, strategies: 8, sum: 'some-sha256-hash', diff --git a/src/lib/services/instance-stats-service.ts b/src/lib/services/instance-stats-service.ts index 70cbe20b19..9116dbacc1 100644 --- a/src/lib/services/instance-stats-service.ts +++ b/src/lib/services/instance-stats-service.ts @@ -33,6 +33,7 @@ export interface InstanceStats { contextFields: number; roles: number; customRootRoles: number; + customRootRolesInUse: number; featureExports: number; featureImports: number; groups: number; @@ -180,6 +181,7 @@ export class InstanceStatsService { groups, roles, customRootRoles, + customRootRolesInUse, environments, segments, strategies, @@ -196,6 +198,7 @@ export class InstanceStatsService { this.groupStore.count(), this.roleStore.count(), this.roleStore.filteredCount({ type: CUSTOM_ROOT_ROLE_TYPE }), + this.roleStore.filteredCountInUse({ type: CUSTOM_ROOT_ROLE_TYPE }), this.environmentStore.count(), this.segmentStore.count(), this.strategyStore.count(), @@ -217,6 +220,7 @@ export class InstanceStatsService { contextFields, roles, customRootRoles, + customRootRolesInUse, groups, environments, segments, diff --git a/src/lib/services/version-service.ts b/src/lib/services/version-service.ts index 84759dd8b7..3b9f6782c6 100644 --- a/src/lib/services/version-service.ts +++ b/src/lib/services/version-service.ts @@ -229,6 +229,7 @@ export default class VersionService { groups, roles, customRootRoles, + customRootRolesInUse, environments, segments, strategies, @@ -248,6 +249,7 @@ export default class VersionService { this.roleStore.filteredCount({ type: CUSTOM_ROOT_ROLE_TYPE, }), + this.roleStore.filteredCountInUse({ type: CUSTOM_ROOT_ROLE_TYPE }), this.environmentStore.count(), this.segmentStore.count(), this.strategyStore.count(), @@ -269,6 +271,7 @@ export default class VersionService { groups, roles, customRootRoles, + customRootRolesInUse, environments, segments, strategies, diff --git a/src/lib/types/stores/role-store.ts b/src/lib/types/stores/role-store.ts index 1bc367aa6c..27ca9b7f6f 100644 --- a/src/lib/types/stores/role-store.ts +++ b/src/lib/types/stores/role-store.ts @@ -31,4 +31,5 @@ export interface IRoleStore extends Store { nameInUse(name: string, existingId?: number): Promise; count(): Promise; filteredCount(filter: Partial): Promise; + filteredCountInUse(filter: Partial): Promise; } diff --git a/src/test/fixtures/fake-role-store.ts b/src/test/fixtures/fake-role-store.ts index bcf22bea67..034d4586c2 100644 --- a/src/test/fixtures/fake-role-store.ts +++ b/src/test/fixtures/fake-role-store.ts @@ -17,6 +17,10 @@ export default class FakeRoleStore implements IRoleStore { return Promise.resolve(0); } + filteredCountInUse(search: Partial): Promise { + return Promise.resolve(0); + } + roles: ICustomRole[] = []; getGroupRolesForProject(projectId: string): Promise {