Mirrors/unleash.unleash
1
0
Fork 0
mirror of https://github.com/Unleash/unleash.git synced 2025-04-06 01:15:28 +02:00
Code Issues Projects Releases Wiki Activity

fix: change .inc calls to .increment (#8000)

Browse Source 
We are observing incorrect data in Prometheus, which is consistently
non-reproducible. After a restart, the issue does not occur, but if the
pods run for an extended period, they seem to enter a strange state
where the counters become entangled and start sharing arbitrary values
that are added to the counters.

For example, the `feature_lifecycle_stage_entered` counter gets an
arbitrary value, such as 12, added when `inc()` is called. The
`exceedsLimitErrorCounter` shows the same behavior, and the code
implementation is identical.

We also tested some existing `increase()` counters, and they do not
suffer from this issue.

All calls to `counter.labels(labels).inc(`) will be replaced by
`counter.increment()` to try to mitigate the issue.
This commit is contained in:
Jaanus Sellin 2024-08-28 12:50:36 +03:00 committed by GitHub
parent bed4f66fa2
commit 7ad686e14e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 8 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/lib/metrics.ts
View File

@ -428,9 +428,9 @@ export default class MetricsMonitor {
eventBus.on( eventBus.on(
events.STAGE_ENTERED, events.STAGE_ENTERED,
(entered: { stage: string; feature: string }) => { (entered: { stage: string; feature: string }) => {
featureLifecycleStageEnteredCounter featureLifecycleStageEnteredCounter.increment({
.labels({ stage: entered.stage }) stage: entered.stage,
.inc(); });
}, },
); );
@ -440,9 +440,7 @@ export default class MetricsMonitor {
resource, resource,
limit, limit,
}: { resource: string; limit: number }) => { }: { resource: string; limit: number }) => {
exceedsLimitErrorCounter exceedsLimitErrorCounter.increment({ resource, limit });
.labels({ resource, limit })
.inc();
}, },
); );
@ -862,16 +860,16 @@ export default class MetricsMonitor {
}); });
}); });
eventStore.on(PROJECT_CREATED, () => { eventStore.on(PROJECT_CREATED, () => {
projectActionsCounter.labels({ action: PROJECT_CREATED }).inc(); projectActionsCounter.increment({ action: PROJECT_CREATED });
}); });
eventStore.on(PROJECT_ARCHIVED, () => { eventStore.on(PROJECT_ARCHIVED, () => {
projectActionsCounter.labels({ action: PROJECT_ARCHIVED }).inc(); projectActionsCounter.increment({ action: PROJECT_ARCHIVED });
}); });
eventStore.on(PROJECT_REVIVED, () => { eventStore.on(PROJECT_REVIVED, () => {
projectActionsCounter.labels({ action: PROJECT_REVIVED }).inc(); projectActionsCounter.increment({ action: PROJECT_REVIVED });
}); });
eventStore.on(PROJECT_DELETED, () => { eventStore.on(PROJECT_DELETED, () => {
projectActionsCounter.labels({ action: PROJECT_DELETED }).inc(); projectActionsCounter.increment({ action: PROJECT_DELETED });
}); });
const logger = config.getLogger('metrics.ts'); const logger = config.getLogger('metrics.ts');