refactor: add functions to estimate monthly usage from data directly (#9219)

Adds new monthly estimation functions that operate on raw usage data
instead of chart data. This brings those methods in line with the rest
of the traffic calculation functions that we have in that file and means
we can remove other external dependencies.

 This is somewhat inspired by #9218, but not directly linked.

frontend/src/component/admin/network/NetworkTrafficUsage/NetworkTrafficUsage.tsx

@@ -28,7 +28,10 @@ import type { Theme } from '@mui/material/styles/createTheme';
 import Grid from '@mui/material/Grid';
 import { NetworkTrafficUsagePlanSummary } from './NetworkTrafficUsagePlanSummary';
 import annotationPlugin from 'chartjs-plugin-annotation';
-import { useTrafficDataEstimation } from 'hooks/useTrafficData';
+import {
+    useTrafficDataEstimation,
+    calculateEstimatedMonthlyCost as deprecatedCalculateEstimatedMonthlyCost,
+} from 'hooks/useTrafficData';
 import { customHighlightPlugin } from 'component/common/Chart/customHighlightPlugin';
 import { formatTickValue } from 'component/common/Chart/formatTickValue';
 import { useTrafficLimit } from './hooks/useTrafficLimit';
@@ -44,7 +47,6 @@ import {
     calculateEstimatedMonthlyCost,
 } from 'utils/traffic-calculations';
 import { currentDate, currentMonth } from './dates';
-import { endpointsInfo } from './endpoint-info';
 import { type ChartDataSelection, toDateRange } from './chart-data-selection';
 import {
     type ChartDatasetType,
@@ -245,10 +247,7 @@ const NewNetworkTrafficUsage: FC = () => {
     );
 
     const estimatedMonthlyCost = calculateEstimatedMonthlyCost(
-        chartDataSelection.grouping === 'daily'
+        traffic.usage?.apiData,
-            ? chartDataSelection.month
-            : currentMonth,
-        data.datasets,
         includedTraffic,
         currentDate,
         BILLING_TRAFFIC_BUNDLE_PRICE,
@@ -301,7 +300,6 @@ const NewNetworkTrafficUsage: FC = () => {
                                         chartDataSelection.grouping === 'daily'
                                             ? usageTotal
                                             : averageTrafficPreviousMonths(
-                                                  Object.keys(endpointsInfo),
                                                   traffic.usage,
                                               )
                                     }
@@ -428,7 +426,7 @@ const OldNetworkTrafficUsage: FC = () => {
                 setOverageCost(calculatedOverageCost);
 
                 setEstimatedMonthlyCost(
-                    calculateEstimatedMonthlyCost(
+                    deprecatedCalculateEstimatedMonthlyCost(
                         period,
                         data.datasets,
                         includedTraffic,

frontend/src/component/admin/network/NetworkTrafficUsage/average-traffic-previous-months.ts

@@ -1,8 +1,8 @@
-import { differenceInCalendarMonths, format } from 'date-fns';
+import { differenceInCalendarMonths } from 'date-fns';
 import type { TrafficUsageDataSegmentedCombinedSchema } from 'openapi';
+import { currentMonth } from './dates';
 
 export const averageTrafficPreviousMonths = (
-    endpointData: string[],
     traffic: TrafficUsageDataSegmentedCombinedSchema,
 ) => {
     if (!traffic || traffic.grouping === 'daily') {
@@ -16,10 +16,7 @@ export const averageTrafficPreviousMonths = (
         ),
     );
 
-    const currentMonth = format(new Date(), 'yyyy-MM');
-
     const totalTraffic = traffic.apiData
-        .filter((endpoint) => endpointData.includes(endpoint.apiPath))
         .map((endpoint) =>
             endpoint.dataPoints
                 .filter(({ period }) => period !== currentMonth)

frontend/src/hooks/useTrafficData.ts

@@ -1,6 +1,15 @@
 import type { ChartDatasetType } from 'component/admin/network/NetworkTrafficUsage/chart-functions';
 import type { IInstanceTrafficMetricsResponse } from './api/getters/useInstanceTrafficMetrics/useInstanceTrafficMetrics';
 import { useState } from 'react';
+import {
+    DEFAULT_TRAFFIC_DATA_UNIT_COST,
+    DEFAULT_TRAFFIC_DATA_UNIT_SIZE,
+    calculateOverageCost,
+} from 'utils/traffic-calculations';
+import {
+    currentMonth,
+    daysInCurrentMonth,
+} from 'component/admin/network/NetworkTrafficUsage/dates';
 
 export type SelectablePeriod = {
     key: string;
@@ -148,6 +157,54 @@ const toTrafficUsageSum = (trafficData: ChartDatasetType[]): number => {
     return data;
 };
 
+export const calculateProjectedUsage = (
+    today: number,
+    trafficData: ChartDatasetType[],
+    daysInPeriod: number,
+) => {
+    if (today < 5) {
+        return 0;
+    }
+
+    const spliceToYesterday = today - 1;
+    const trafficDataUpToYesterday = trafficData.map((item) => {
+        return {
+            ...item,
+            data: item.data.slice(0, spliceToYesterday),
+        };
+    });
+
+    const dataUsage = toTrafficUsageSum(trafficDataUpToYesterday);
+    return (dataUsage / spliceToYesterday) * daysInPeriod;
+};
+
+export const calculateEstimatedMonthlyCost = (
+    period: string,
+    trafficData: ChartDatasetType[],
+    includedTraffic: number,
+    currentDate: Date,
+    trafficUnitCost = DEFAULT_TRAFFIC_DATA_UNIT_COST,
+    trafficUnitSize = DEFAULT_TRAFFIC_DATA_UNIT_SIZE,
+) => {
+    if (period !== currentMonth) {
+        return 0;
+    }
+
+    const today = currentDate.getDate();
+    const projectedUsage = calculateProjectedUsage(
+        today,
+        trafficData,
+        daysInCurrentMonth,
+    );
+
+    return calculateOverageCost(
+        projectedUsage,
+        includedTraffic,
+        trafficUnitCost,
+        trafficUnitSize,
+    );
+};
+
 const getDayLabels = (dayCount: number): number[] => {
     return [...Array(dayCount).keys()].map((i) => i + 1);
 };

frontend/src/utils/traffic-calculations.test.ts

@@ -1,4 +1,4 @@
-import { getDaysInMonth } from 'date-fns';
+import { format, getDaysInMonth } from 'date-fns';
 import {
     calculateEstimatedMonthlyCost,
     calculateOverageCost,
@@ -7,7 +7,14 @@ import {
     cleanTrafficData,
 } from './traffic-calculations';
 import { toSelectablePeriod } from '../component/admin/network/NetworkTrafficUsage/selectable-periods';
-import type { TrafficUsageDataSegmentedCombinedSchema } from 'openapi';
+import type {
+    TrafficUsageDataSegmentedCombinedSchema,
+    TrafficUsageDataSegmentedCombinedSchemaApiDataItem,
+} from 'openapi';
+import {
+    calculateEstimatedMonthlyCost as deprecatedCalculateEstimatedMonthlyCost,
+    calculateProjectedUsage as deprecatedCalculateProjectedUsage,
+} from 'hooks/useTrafficData';
 
 const testData4Days = [
     {
@@ -30,6 +37,32 @@ const testData4Days = [
     },
 ];
 
+const dataPoint = (month: Date) => (day: number, count: number) => {
+    const monthPrefix = format(month, 'yyyy-MM');
+
+    return {
+        period: `${monthPrefix}-${day.toString().padStart(2, '0')}`,
+        trafficTypes: [{ count, group: 'successful-requests' }],
+    };
+};
+
+const trafficData4Days = (
+    month: Date,
+): TrafficUsageDataSegmentedCombinedSchemaApiDataItem[] => {
+    const point = dataPoint(month);
+    const dataPoints = [
+        point(1, 23_000_000),
+        point(2, 22_000_000),
+        point(3, 24_000_000),
+        point(4, 21_000_000),
+    ];
+    return [
+        { apiPath: '/api/frontend', dataPoints },
+        { apiPath: '/api/admin', dataPoints },
+        { apiPath: '/api/client', dataPoints },
+    ];
+};
+
 describe('traffic overage calculation', () => {
     it('should return 0 if there is no overage this month', () => {
         const dataUsage = 52_900_000;
@@ -63,13 +96,22 @@ describe('traffic overage calculation', () => {
         const now = new Date();
         const period = toSelectablePeriod(now);
         const testNow = new Date(now.getFullYear(), now.getMonth(), 4);
-        const result = calculateEstimatedMonthlyCost(
+        const includedTraffic = 53_000_000;
+        const result = deprecatedCalculateEstimatedMonthlyCost(
             period.key,
             testData4Days,
-            53_000_000,
+            includedTraffic,
             testNow,
         );
         expect(result).toBe(0);
+
+        const rawData = trafficData4Days(now);
+        const result2 = calculateEstimatedMonthlyCost(
+            rawData,
+            includedTraffic,
+            testNow,
+        );
+        expect(result2).toBe(result);
     });
 
     it('needs 5 days or more to estimate for the month', () => {
@@ -80,13 +122,25 @@ describe('traffic overage calculation', () => {
         const now = new Date();
         const period = toSelectablePeriod(now);
         const testNow = new Date(now.getFullYear(), now.getMonth(), 5);
-        const result = calculateEstimatedMonthlyCost(
+        const includedTraffic = 53_000_000;
+        const result = deprecatedCalculateEstimatedMonthlyCost(
             period.key,
             testData,
-            53_000_000,
+            includedTraffic,
             testNow,
         );
         expect(result).toBeGreaterThan(1430);
+
+        const rawData = trafficData4Days(now);
+        rawData[0].dataPoints.push(dataPoint(now)(5, 23_000_000));
+        rawData[1].dataPoints.push(dataPoint(now)(5, 23_000_000));
+        rawData[2].dataPoints.push(dataPoint(now)(5, 23_000_000));
+        const result2 = calculateEstimatedMonthlyCost(
+            rawData,
+            includedTraffic,
+            testNow,
+        );
+        expect(result2).toBe(result);
     });
 
     it('estimates projected data usage', () => {
@@ -97,13 +151,34 @@ describe('traffic overage calculation', () => {
         // Testing April 5th of 2024 (30 days)
         const now = new Date(2024, 3, 5);
         const period = toSelectablePeriod(now);
-        const result = calculateProjectedUsage(
+        const result = deprecatedCalculateProjectedUsage(
             now.getDate(),
             testData,
             period.dayCount,
         );
         // 22_500_000 * 3 * 30 = 2_025_000_000
         expect(result).toBe(2_025_000_000);
+
+        const rawData = trafficData4Days(now);
+        rawData[0].dataPoints.push(dataPoint(now)(5, 22_500_000));
+        rawData[1].dataPoints.push(dataPoint(now)(5, 22_500_000));
+        rawData[2].dataPoints.push(dataPoint(now)(5, 22_500_000));
+        const result2 = calculateProjectedUsage({
+            dayOfMonth: now.getDate(),
+            daysInMonth: period.dayCount,
+            trafficData: rawData,
+        });
+        expect(result2).toBe(result);
+    });
+
+    it("doesn't die if traffic is undefined", () => {
+        expect(
+            calculateEstimatedMonthlyCost(
+                undefined,
+                500_000,
+                new Date('2024-05-15'),
+            ),
+        ).toBe(0);
     });
 
     it('supports custom price and unit size', () => {
@@ -129,7 +204,7 @@ describe('traffic overage calculation', () => {
         const includedTraffic = 53_000_000;
         const trafficUnitSize = 500_000;
         const trafficUnitCost = 10;
-        const result = calculateEstimatedMonthlyCost(
+        const result = deprecatedCalculateEstimatedMonthlyCost(
             period.key,
             testData,
             includedTraffic,
@@ -143,6 +218,20 @@ describe('traffic overage calculation', () => {
         const overageUnits = Math.floor(overage / trafficUnitSize);
         const total = overageUnits * trafficUnitCost;
         expect(result).toBe(total);
+
+        const rawData = trafficData4Days(now);
+        rawData[0].dataPoints.push(dataPoint(now)(5, 22_500_000));
+        rawData[1].dataPoints.push(dataPoint(now)(5, 22_500_000));
+        rawData[2].dataPoints.push(dataPoint(now)(5, 22_500_000));
+        const result2 = calculateEstimatedMonthlyCost(
+            rawData,
+            includedTraffic,
+            testNow,
+            trafficUnitCost,
+            trafficUnitSize,
+        );
+
+        expect(result2).toBe(result);
     });
 });
 

frontend/src/utils/traffic-calculations.ts

@@ -2,15 +2,10 @@ import type {
     TrafficUsageDataSegmentedCombinedSchema,
     TrafficUsageDataSegmentedCombinedSchemaApiDataItem,
 } from 'openapi';
-import {
+import { getDaysInMonth } from 'date-fns';
-    currentMonth,
-    daysInCurrentMonth,
-} from '../component/admin/network/NetworkTrafficUsage/dates';
-import type { ChartDatasetType } from '../component/admin/network/NetworkTrafficUsage/chart-functions';
 import { format } from 'date-fns';
-
+export const DEFAULT_TRAFFIC_DATA_UNIT_COST = 5;
-const DEFAULT_TRAFFIC_DATA_UNIT_COST = 5;
+export const DEFAULT_TRAFFIC_DATA_UNIT_SIZE = 1_000_000;
-const DEFAULT_TRAFFIC_DATA_UNIT_SIZE = 1_000_000;
 
 export const TRAFFIC_MEASUREMENT_START_DATE = new Date('2024-05-01');
 
@@ -112,61 +107,53 @@ export const calculateOverageCost = (
         : 0;
 };
 
-export const calculateProjectedUsage = (
+export const calculateProjectedUsage = ({
-    today: number,
+    dayOfMonth,
-    trafficData: ChartDatasetType[],
+    daysInMonth,
-    daysInPeriod: number,
+    trafficData,
-) => {
+}: {
-    if (today < 5) {
+    dayOfMonth: number;
+    daysInMonth: number;
+    trafficData: TrafficUsageDataSegmentedCombinedSchemaApiDataItem[];
+}) => {
+    if (dayOfMonth < 5) {
         return 0;
     }
 
-    const spliceToYesterday = today - 1;
     const trafficDataUpToYesterday = trafficData.map((item) => {
         return {
             ...item,
-            data: item.data.slice(0, spliceToYesterday),
+            dataPoints: item.dataPoints.filter(
+                (point) => Number(point.period.slice(-2)) < dayOfMonth,
+            ),
         };
     });
 
-    const toTrafficUsageSum = (trafficData: ChartDatasetType[]): number => {
+    const dataUsage = dailyTrafficDataToCurrentUsage(trafficDataUpToYesterday);
-        const data = trafficData.reduce(
-            (acc: number, current: ChartDatasetType) => {
-                return (
-                    acc +
-                    current.data.reduce(
-                        (acc_inner, current_inner) => acc_inner + current_inner,
-                        0,
-                    )
-                );
-            },
-            0,
-        );
-        return data;
-    };
 
-    const dataUsage = toTrafficUsageSum(trafficDataUpToYesterday);
+    return (dataUsage / (dayOfMonth - 1)) * daysInMonth;
-    return (dataUsage / spliceToYesterday) * daysInPeriod;
 };
 
 export const calculateEstimatedMonthlyCost = (
-    period: string,
+    trafficData:
-    trafficData: ChartDatasetType[],
+        | TrafficUsageDataSegmentedCombinedSchemaApiDataItem[]
+        | undefined,
     includedTraffic: number,
     currentDate: Date,
     trafficUnitCost = DEFAULT_TRAFFIC_DATA_UNIT_COST,
     trafficUnitSize = DEFAULT_TRAFFIC_DATA_UNIT_SIZE,
 ) => {
-    if (period !== currentMonth) {
+    if (!trafficData) {
         return 0;
     }
+    const dayOfMonth = currentDate.getDate();
+    const daysInMonth = getDaysInMonth(currentDate);
 
-    const today = currentDate.getDate();
+    const projectedUsage = calculateProjectedUsage({
-    const projectedUsage = calculateProjectedUsage(
+        dayOfMonth,
-        today,
+        daysInMonth,
         trafficData,
-        daysInCurrentMonth,
+    });
-    );
 
     return calculateOverageCost(
         projectedUsage,