feat: fill metrics summary missing datapoints with 0 (#6820)

Fills missing datapoints with 0s so that all metrics chart lines have
data for all datapoints.

Closes #
[1-2278](https://linear.app/unleash/issue/1-2278/fill-the-metrics-data-with-0s-when-not-enough-data-to-fill-the-chart)


Before: 
<img width="1547" alt="Screenshot 2024-04-10 at 12 48 22"
src="https://github.com/Unleash/unleash/assets/104830839/35885852-d986-4760-84e2-9e8ef61bedf0">
<img width="1550" alt="Screenshot 2024-04-10 at 12 48 44"
src="https://github.com/Unleash/unleash/assets/104830839/3385b8eb-08e2-4cc9-86b2-7b31b9fe81ef">

After:
<img width="1582" alt="Screenshot 2024-04-10 at 13 43 10"
src="https://github.com/Unleash/unleash/assets/104830839/d3713df3-869b-48ba-b2ab-095027b37506">
<img width="1545" alt="Screenshot 2024-04-10 at 13 42 49"
src="https://github.com/Unleash/unleash/assets/104830839/44a6e662-2e9f-4fe8-8299-c15ab8f8e261">

---------

Signed-off-by: andreas-unleash <andreas@getunleash.ai>

frontend/src/component/insights/InsightsCharts.tsx

@@ -49,6 +49,7 @@ interface IChartsProps {
     };
     loading: boolean;
     projects: string[];
+    allMetricsDatapoints: string[];
 }
 
 const StyledGrid = styled(Box)(({ theme }) => ({
@@ -79,6 +80,7 @@ export const InsightsCharts: VFC<IChartsProps> = ({
     flagTrends,
     groupedMetricsData,
     environmentTypeTrends,
+    allMetricsDatapoints,
     loading,
 }) => {
     const showAllProjects = projects[0] === allOption.id;
@@ -203,6 +205,7 @@ export const InsightsCharts: VFC<IChartsProps> = ({
             >
                 <MetricsSummaryChart
                     metricsSummaryTrends={groupedMetricsData}
+                    allDatapointsSorted={allMetricsDatapoints}
                     isAggregate={showAllProjects}
                 />
             </Widget>

frontend/src/component/insights/componentsChart/MetricsSummaryChart/MetricsSummaryChart.tsx

@@ -8,25 +8,30 @@ import {
     NotEnoughData,
 } from 'component/insights/components/LineChart/LineChart';
 import { MetricsSummaryTooltip } from './MetricsChartTooltip/MetricsChartTooltip';
-import { useMetricsSummary } from 'component/insights/hooks/useMetricsSummary';
 import { usePlaceholderData } from 'component/insights/hooks/usePlaceholderData';
 import type { GroupedDataByProject } from 'component/insights/hooks/useGroupedProjectTrends';
 import { useTheme } from '@mui/material';
-import { aggregateDataPerDate } from './MetricsChartTooltip/aggregate-metrics-by-day';
+import { aggregateDataPerDate } from './aggregate-metrics-by-day';
+import { useFilledMetricsSummary } from '../../hooks/useFilledMetricsSummary';
 
 interface IMetricsSummaryChartProps {
     metricsSummaryTrends: GroupedDataByProject<
         InstanceInsightsSchema['metricsSummaryTrends']
     >;
     isAggregate?: boolean;
+    allDatapointsSorted: string[];
 }
 
 export const MetricsSummaryChart: VFC<IMetricsSummaryChartProps> = ({
     metricsSummaryTrends,
     isAggregate,
+    allDatapointsSorted,
 }) => {
     const theme = useTheme();
-    const metricsSummary = useMetricsSummary(metricsSummaryTrends);
+    const metricsSummary = useFilledMetricsSummary(
+        metricsSummaryTrends,
+        allDatapointsSorted,
+    );
     const notEnoughData = useMemo(
         () => !metricsSummary.datasets.some((d) => d.data.length > 1),
         [metricsSummary],

frontend/src/component/insights/hooks/useAllDatapoints.ts

@@ -0,0 +1,12 @@
+import { useMemo } from 'react';
+
+export const useAllDatapoints = <T extends { date: string }>(data: T[]) =>
+    useMemo(() => {
+        const allDataPoints = new Set<string>();
+
+        data.forEach((item) => {
+            allDataPoints.add(item.date);
+        });
+
+        return Array.from(allDataPoints).sort();
+    }, [data]);

frontend/src/component/insights/hooks/useFilledMetricsSummary.test.ts

@@ -0,0 +1,54 @@
+import { renderHook } from '@testing-library/react-hooks';
+import { useTheme } from '@mui/material';
+import { useProjectColor } from './useProjectColor';
+import { useFilledMetricsSummary } from './useFilledMetricsSummary';
+import type { Theme } from '@mui/material/styles';
+import type { InstanceInsightsSchema } from '../../../openapi';
+import type { GroupedDataByProject } from './useGroupedProjectTrends';
+import { vi, type Mock } from 'vitest';
+
+vi.mock('@mui/material', () => ({
+    useTheme: vi.fn(),
+}));
+
+vi.mock('./useProjectColor', () => ({
+    useProjectColor: vi.fn(),
+}));
+
+// Mock data
+const mockTheme: Partial<Theme> = {};
+const mockGetProjectColor = (project: string) => `color-${project}`;
+const mockFilteredMetricsSummaryTrends = {
+    Project1: [{ date: '2024-01-01', totalRequests: 5 }],
+    Project2: [{ date: '2024-01-02', totalRequests: 10 }],
+} as unknown as GroupedDataByProject<
+    InstanceInsightsSchema['metricsSummaryTrends']
+>;
+const mockAllDataPointsSorted = ['2024-01-01', '2024-01-02'];
+
+beforeEach(() => {
+    (useTheme as Mock).mockReturnValue(mockTheme);
+    (useProjectColor as Mock).mockImplementation(() => mockGetProjectColor);
+});
+
+describe('useFilledMetricsSummary', () => {
+    it('returns datasets with normalized data for each project', () => {
+        const { result } = renderHook(() =>
+            useFilledMetricsSummary(
+                mockFilteredMetricsSummaryTrends,
+                mockAllDataPointsSorted,
+            ),
+        );
+
+        expect(result.current.datasets).toHaveLength(2); // Expect two projects
+        expect(result.current.datasets[0].data).toHaveLength(2); // Each dataset should have data for both dates
+
+        // Check for normalized missing data
+        const project1DataFor20240102 = result.current.datasets
+            .find((dataset) => dataset.label === 'Project1')
+            ?.data.find((data) => data.date === '2024-01-02');
+        expect(project1DataFor20240102).toEqual(
+            expect.objectContaining({ totalRequests: 0 }),
+        ); // Missing data should be filled with 0
+    });
+});

frontend/src/component/insights/hooks/useFilledMetricsSummary.ts

@@ -0,0 +1,63 @@
+import { useMemo } from 'react';
+import { useTheme } from '@mui/material';
+import type {
+    InstanceInsightsSchema,
+    InstanceInsightsSchemaMetricsSummaryTrendsItem,
+} from 'openapi';
+import { useProjectColor } from './useProjectColor';
+import type { GroupedDataByProject } from './useGroupedProjectTrends';
+import { format } from 'date-fns';
+
+type MetricsSummaryTrends = InstanceInsightsSchema['metricsSummaryTrends'];
+
+const weekIdFromDate = (dateString: string) => {
+    return format(new Date(dateString), 'yyyy-ww');
+};
+
+export const useFilledMetricsSummary = (
+    filteredMetricsSummaryTrends: GroupedDataByProject<MetricsSummaryTrends>,
+    allDataPointsSorted: string[],
+) => {
+    const theme = useTheme();
+    const getProjectColor = useProjectColor();
+
+    const data = useMemo(() => {
+        const datasets = Object.entries(filteredMetricsSummaryTrends).map(
+            ([project, trends]) => {
+                const trendsMap = new Map<
+                    string,
+                    InstanceInsightsSchemaMetricsSummaryTrendsItem
+                >(trends.map((trend) => [trend.date, trend]));
+
+                const normalizedData = allDataPointsSorted.map((date) => {
+                    return (
+                        trendsMap.get(date) || {
+                            date,
+                            totalRequests: 0,
+                            totalNo: 0,
+                            project,
+                            totalApps: 0,
+                            totalYes: 0,
+                            totalEnvironments: 0,
+                            totalFlags: 0,
+                            week: weekIdFromDate(date),
+                        }
+                    );
+                });
+
+                const color = getProjectColor(project);
+                return {
+                    label: project,
+                    data: normalizedData,
+                    borderColor: color,
+                    backgroundColor: color,
+                    fill: false,
+                };
+            },
+        );
+
+        return { datasets };
+    }, [theme, filteredMetricsSummaryTrends, getProjectColor]);
+
+    return data;
+};

frontend/src/component/insights/hooks/useInsightsData.ts

@@ -3,42 +3,47 @@ import type { InstanceInsightsSchema } from 'openapi';
 import { useFilteredTrends } from './useFilteredTrends';
 import { useGroupedProjectTrends } from './useGroupedProjectTrends';
 import { useFilteredFlagsSummary } from './useFilteredFlagsSummary';
+import { useAllDatapoints } from './useAllDatapoints';
 
 export const useInsightsData = (
-    executiveDashboardData: InstanceInsightsSchema,
+    instanceInsights: InstanceInsightsSchema,
     projects: string[],
 ) => {
+    const allMetricsDatapoints = useAllDatapoints(
+        instanceInsights.metricsSummaryTrends,
+    );
     const projectsData = useFilteredTrends(
-        executiveDashboardData.projectFlagTrends,
+        instanceInsights.projectFlagTrends,
         projects,
     );
 
     const groupedProjectsData = useGroupedProjectTrends(projectsData);
 
     const metricsData = useFilteredTrends(
-        executiveDashboardData.metricsSummaryTrends,
+        instanceInsights.metricsSummaryTrends,
         projects,
     );
     const groupedMetricsData = useGroupedProjectTrends(metricsData);
 
     const summary = useFilteredFlagsSummary(
         projectsData,
-        executiveDashboardData.users,
+        instanceInsights.users,
     );
 
     return useMemo(
         () => ({
-            ...executiveDashboardData,
+            ...instanceInsights,
             projectsData,
             groupedProjectsData,
             metricsData,
             groupedMetricsData,
-            users: executiveDashboardData.users,
+            users: instanceInsights.users,
-            environmentTypeTrends: executiveDashboardData.environmentTypeTrends,
+            environmentTypeTrends: instanceInsights.environmentTypeTrends,
             summary,
+            allMetricsDatapoints,
         }),
         [
-            executiveDashboardData,
+            instanceInsights,
             projects,
             projectsData,
             groupedProjectsData,

frontend/src/component/insights/hooks/useMetricsSummary.ts

@@ -1,32 +0,0 @@
-import { useMemo } from 'react';
-import { useTheme } from '@mui/material';
-import type { InstanceInsightsSchema } from 'openapi';
-import { useProjectColor } from './useProjectColor';
-import type { GroupedDataByProject } from './useGroupedProjectTrends';
-
-type MetricsSummaryTrends = InstanceInsightsSchema['metricsSummaryTrends'];
-
-export const useMetricsSummary = (
-    metricsSummaryTrends: GroupedDataByProject<MetricsSummaryTrends>,
-) => {
-    const theme = useTheme();
-    const getProjectColor = useProjectColor();
-
-    const data = useMemo(() => {
-        const datasets = Object.entries(metricsSummaryTrends).map(
-            ([project, trends]) => {
-                const color = getProjectColor(project);
-                return {
-                    label: project,
-                    data: trends,
-                    borderColor: color,
-                    backgroundColor: color,
-                    fill: false,
-                };
-            },
-        );
-        return { datasets };
-    }, [theme, metricsSummaryTrends]);
-
-    return data;
-};