Add graph showing motion and object activity to history timeline desktop view (#9184)

* Add timeline graph component

* Add more custom colors and improve graph

* Add api and data

* Fix data sorting

* Add graph to timeline

* Only show timeline for selected hour

* Make data full range

frigate/http.py

@@ -8,6 +8,7 @@ import re
 import subprocess as sp
 import time
 import traceback
+from collections import defaultdict
 from datetime import datetime, timedelta, timezone
 from functools import reduce
 from pathlib import Path
@@ -620,7 +621,9 @@ def hourly_timeline():
     after = request.args.get("after", type=float)
     limit = request.args.get("limit", 200)
     tz_name = request.args.get("timezone", default="utc", type=str)
+
     _, minute_modifier, _ = get_tz_modifiers(tz_name)
+    minute_offset = int(minute_modifier.split(" ")[0])
 
     clauses = []
 
@@ -675,7 +678,7 @@ def hourly_timeline():
                 minute=0, second=0, microsecond=0
             )
             + timedelta(
-                minutes=int(minute_modifier.split(" ")[0]),
+                minutes=minute_offset,
             )
         ).timestamp()
         if hour not in hours:
@@ -693,6 +696,87 @@ def hourly_timeline():
     )
 
 
+@bp.route("/<camera_name>/recording/hourly/activity")
+def hourly_timeline_activity(camera_name: str):
+    """Get hourly summary for timeline."""
+    if camera_name not in current_app.frigate_config.cameras:
+        return make_response(
+            jsonify({"success": False, "message": "Camera not found"}),
+            404,
+        )
+
+    before = request.args.get("before", type=float, default=datetime.now())
+    after = request.args.get(
+        "after", type=float, default=datetime.now() - timedelta(hours=1)
+    )
+    tz_name = request.args.get("timezone", default="utc", type=str)
+
+    _, minute_modifier, _ = get_tz_modifiers(tz_name)
+    minute_offset = int(minute_modifier.split(" ")[0])
+
+    all_recordings: list[Recordings] = (
+        Recordings.select(
+            Recordings.start_time,
+            Recordings.duration,
+            Recordings.objects,
+            Recordings.motion,
+        )
+        .where(Recordings.camera == camera_name)
+        .where(Recordings.motion > 0)
+        .where((Recordings.start_time > after) & (Recordings.end_time < before))
+        .order_by(Recordings.start_time.asc())
+        .iterator()
+    )
+
+    # data format is ex:
+    # {timestamp: [{ date: 1, count: 1, type: motion }]}] }}
+    hours: dict[int, list[dict[str, any]]] = defaultdict(list)
+
+    key = datetime.fromtimestamp(after).replace(second=0, microsecond=0) + timedelta(
+        minutes=minute_offset
+    )
+    check = (key + timedelta(hours=1)).timestamp()
+
+    # set initial start so data is representative of full hour
+    hours[int(key.timestamp())].append(
+        {
+            "date": key.timestamp(),
+            "count": 0,
+            "type": "motion",
+        }
+    )
+
+    for recording in all_recordings:
+        if recording.start_time > check:
+            hours[int(key.timestamp())].append(
+                {
+                    "date": (key + timedelta(hours=1)).timestamp(),
+                    "count": 0,
+                    "type": "motion",
+                }
+            )
+            key = key + timedelta(hours=1)
+            check = (key + timedelta(hours=1)).timestamp()
+            hours[int(key.timestamp())].append(
+                {
+                    "date": key.timestamp(),
+                    "count": 0,
+                    "type": "motion",
+                }
+            )
+
+        data_type = "motion" if recording.objects == 0 else "objects"
+        hours[int(key.timestamp())].append(
+            {
+                "date": recording.start_time + (recording.duration / 2),
+                "count": recording.motion,
+                "type": data_type,
+            }
+        )
+
+    return jsonify(hours)
+
+
 @bp.route("/<camera_name>/<label>/best.jpg")
 @bp.route("/<camera_name>/<label>/thumbnail.jpg")
 def label_thumbnail(camera_name, label):

web/package-lock.json

@@ -24,6 +24,7 @@
         "@radix-ui/react-switch": "^1.0.3",
         "@radix-ui/react-tabs": "^1.0.4",
         "@radix-ui/react-tooltip": "^1.0.7",
+        "apexcharts": "^3.45.1",
         "axios": "^1.6.2",
         "class-variance-authority": "^0.7.0",
         "clsx": "^2.0.0",
@@ -34,6 +35,7 @@
         "lucide-react": "^0.294.0",
         "monaco-yaml": "^5.1.0",
         "react": "^18.2.0",
+        "react-apexcharts": "^1.4.1",
         "react-day-picker": "^8.9.1",
         "react-dom": "^18.2.0",
         "react-hook-form": "^7.48.2",
@@ -2782,6 +2784,11 @@
         "node": ">=10.0.0"
       }
     },
+    "node_modules/@yr/monotone-cubic-spline": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/@yr/monotone-cubic-spline/-/monotone-cubic-spline-1.0.3.tgz",
+      "integrity": "sha512-FQXkOta0XBSUPHndIKON2Y9JeQz5ZeMqLYZVVK93FliNBFm7LNMIZmY6FrMEB9XPcDbE2bekMbZD6kzDkxwYjA=="
+    },
     "node_modules/acorn": {
       "version": "8.11.2",
       "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.2.tgz",
@@ -2933,6 +2940,20 @@
         "node": ">= 8"
       }
     },
+    "node_modules/apexcharts": {
+      "version": "3.45.1",
+      "resolved": "https://registry.npmjs.org/apexcharts/-/apexcharts-3.45.1.tgz",
+      "integrity": "sha512-pPjj/SA6dfPvR/IKRZF0STdfBGpBh3WRt7K0DFuW9P8erypYkX17EHu3/molPRfo2zSiQwTVpshHC5ncysqfkA==",
+      "dependencies": {
+        "@yr/monotone-cubic-spline": "^1.0.3",
+        "svg.draggable.js": "^2.2.2",
+        "svg.easing.js": "^2.0.0",
+        "svg.filter.js": "^2.0.2",
+        "svg.pathmorphing.js": "^0.1.3",
+        "svg.resize.js": "^1.4.3",
+        "svg.select.js": "^3.0.1"
+      }
+    },
     "node_modules/arg": {
       "version": "5.0.2",
       "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz",
@@ -6341,6 +6362,21 @@
         "node": ">= 0.6.0"
       }
     },
+    "node_modules/prop-types": {
+      "version": "15.8.1",
+      "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz",
+      "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==",
+      "dependencies": {
+        "loose-envify": "^1.4.0",
+        "object-assign": "^4.1.1",
+        "react-is": "^16.13.1"
+      }
+    },
+    "node_modules/prop-types/node_modules/react-is": {
+      "version": "16.13.1",
+      "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
+      "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ=="
+    },
     "node_modules/propagating-hammerjs": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/propagating-hammerjs/-/propagating-hammerjs-2.0.1.tgz",
@@ -6406,6 +6442,18 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/react-apexcharts": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/react-apexcharts/-/react-apexcharts-1.4.1.tgz",
+      "integrity": "sha512-G14nVaD64Bnbgy8tYxkjuXEUp/7h30Q0U33xc3AwtGFijJB9nHqOt1a6eG0WBn055RgRg+NwqbKGtqPxy15d0Q==",
+      "dependencies": {
+        "prop-types": "^15.8.1"
+      },
+      "peerDependencies": {
+        "apexcharts": "^3.41.0",
+        "react": ">=0.13"
+      }
+    },
     "node_modules/react-day-picker": {
       "version": "8.9.1",
       "resolved": "https://registry.npmjs.org/react-day-picker/-/react-day-picker-8.9.1.tgz",
@@ -7270,6 +7318,89 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/svg.draggable.js": {
+      "version": "2.2.2",
+      "resolved": "https://registry.npmjs.org/svg.draggable.js/-/svg.draggable.js-2.2.2.tgz",
+      "integrity": "sha512-JzNHBc2fLQMzYCZ90KZHN2ohXL0BQJGQimK1kGk6AvSeibuKcIdDX9Kr0dT9+UJ5O8nYA0RB839Lhvk4CY4MZw==",
+      "dependencies": {
+        "svg.js": "^2.0.1"
+      },
+      "engines": {
+        "node": ">= 0.8.0"
+      }
+    },
+    "node_modules/svg.easing.js": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/svg.easing.js/-/svg.easing.js-2.0.0.tgz",
+      "integrity": "sha512-//ctPdJMGy22YoYGV+3HEfHbm6/69LJUTAqI2/5qBvaNHZ9uUFVC82B0Pl299HzgH13rKrBgi4+XyXXyVWWthA==",
+      "dependencies": {
+        "svg.js": ">=2.3.x"
+      },
+      "engines": {
+        "node": ">= 0.8.0"
+      }
+    },
+    "node_modules/svg.filter.js": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/svg.filter.js/-/svg.filter.js-2.0.2.tgz",
+      "integrity": "sha512-xkGBwU+dKBzqg5PtilaTb0EYPqPfJ9Q6saVldX+5vCRy31P6TlRCP3U9NxH3HEufkKkpNgdTLBJnmhDHeTqAkw==",
+      "dependencies": {
+        "svg.js": "^2.2.5"
+      },
+      "engines": {
+        "node": ">= 0.8.0"
+      }
+    },
+    "node_modules/svg.js": {
+      "version": "2.7.1",
+      "resolved": "https://registry.npmjs.org/svg.js/-/svg.js-2.7.1.tgz",
+      "integrity": "sha512-ycbxpizEQktk3FYvn/8BH+6/EuWXg7ZpQREJvgacqn46gIddG24tNNe4Son6omdXCnSOaApnpZw6MPCBA1dODA=="
+    },
+    "node_modules/svg.pathmorphing.js": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/svg.pathmorphing.js/-/svg.pathmorphing.js-0.1.3.tgz",
+      "integrity": "sha512-49HWI9X4XQR/JG1qXkSDV8xViuTLIWm/B/7YuQELV5KMOPtXjiwH4XPJvr/ghEDibmLQ9Oc22dpWpG0vUDDNww==",
+      "dependencies": {
+        "svg.js": "^2.4.0"
+      },
+      "engines": {
+        "node": ">= 0.8.0"
+      }
+    },
+    "node_modules/svg.resize.js": {
+      "version": "1.4.3",
+      "resolved": "https://registry.npmjs.org/svg.resize.js/-/svg.resize.js-1.4.3.tgz",
+      "integrity": "sha512-9k5sXJuPKp+mVzXNvxz7U0uC9oVMQrrf7cFsETznzUDDm0x8+77dtZkWdMfRlmbkEEYvUn9btKuZ3n41oNA+uw==",
+      "dependencies": {
+        "svg.js": "^2.6.5",
+        "svg.select.js": "^2.1.2"
+      },
+      "engines": {
+        "node": ">= 0.8.0"
+      }
+    },
+    "node_modules/svg.resize.js/node_modules/svg.select.js": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/svg.select.js/-/svg.select.js-2.1.2.tgz",
+      "integrity": "sha512-tH6ABEyJsAOVAhwcCjF8mw4crjXSI1aa7j2VQR8ZuJ37H2MBUbyeqYr5nEO7sSN3cy9AR9DUwNg0t/962HlDbQ==",
+      "dependencies": {
+        "svg.js": "^2.2.5"
+      },
+      "engines": {
+        "node": ">= 0.8.0"
+      }
+    },
+    "node_modules/svg.select.js": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/svg.select.js/-/svg.select.js-3.0.1.tgz",
+      "integrity": "sha512-h5IS/hKkuVCbKSieR9uQCj9w+zLHoPh+ce19bBYyqF53g6mnPB8sAtIbe1s9dh2S2fCmYX2xel1Ln3PJBbK4kw==",
+      "dependencies": {
+        "svg.js": "^2.6.5"
+      },
+      "engines": {
+        "node": ">= 0.8.0"
+      }
+    },
     "node_modules/swr": {
       "version": "2.2.4",
       "resolved": "https://registry.npmjs.org/swr/-/swr-2.2.4.tgz",

web/package.json

@@ -29,6 +29,7 @@
     "@radix-ui/react-switch": "^1.0.3",
     "@radix-ui/react-tabs": "^1.0.4",
     "@radix-ui/react-tooltip": "^1.0.7",
+    "apexcharts": "^3.45.1",
     "axios": "^1.6.2",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.0.0",
@@ -39,6 +40,7 @@
     "lucide-react": "^0.294.0",
     "monaco-yaml": "^5.1.0",
     "react": "^18.2.0",
+    "react-apexcharts": "^1.4.1",
     "react-day-picker": "^8.9.1",
     "react-dom": "^18.2.0",
     "react-hook-form": "^7.48.2",

web/src/components/camera/DynamicCameraImage.tsx

@@ -83,19 +83,19 @@ export default function DynamicCameraImage({
       <div className="flex absolute right-0 bottom-0 bg-black bg-opacity-20 rounded p-1">
         <MdLeakAdd
           className={`${
-            detectingMotion == "ON" ? "text-red-500" : "text-gray-600"
+            detectingMotion == "ON" ? "text-motion" : "text-gray-600"
           }`}
         />
         <TbUserScan
           className={`${
-            activeObjects.length > 0 ? "text-cyan-500" : "text-gray-600"
+            activeObjects.length > 0 ? "text-object" : "text-gray-600"
           }`}
         />
         {camera.audio.enabled && (
           <LuEar
             className={`${
               parseInt(audioRms) >= camera.audio.min_volume
-                ? "text-orange-500"
+                ? "text-audio"
                 : "text-gray-600"
             }`}
           />

web/src/components/card/HistoryCard.tsx

@@ -59,7 +59,7 @@ export default function HistoryCard({
           </div>
           <Button className="px-2 py-2" variant="ghost" size="xs">
             <LuTrash
-              className="w-5 h-5 stroke-red-500"
+              className="w-5 h-5 stroke-danger"
               onClick={(e: Event) => {
                 e.stopPropagation();
 

web/src/components/graph/TimelineGraph.tsx

@@ -0,0 +1,65 @@
+import { GraphData } from "@/types/graph";
+import Chart from "react-apexcharts";
+
+type TimelineGraphProps = {
+  id: string;
+  data: GraphData[];
+};
+
+/**
+ * A graph meant to be overlaid on top of a timeline
+ */
+export default function TimelineGraph({ id, data }: TimelineGraphProps) {
+  return (
+    <Chart
+      type="bar"
+      options={{
+        colors: ["#991b1b", "#06b6d4", "#ea580c"],
+        chart: {
+          id: id,
+          selection: {
+            enabled: false,
+          },
+          toolbar: {
+            show: false,
+          },
+          zoom: {
+            enabled: false,
+          },
+        },
+        dataLabels: { enabled: false },
+        grid: {
+          show: false,
+        },
+        legend: {
+          show: false,
+          position: "top",
+        },
+        tooltip: {
+          enabled: false,
+        },
+        xaxis: {
+          type: "datetime",
+          axisBorder: {
+            show: false,
+          },
+          axisTicks: {
+            show: false,
+          },
+          labels: {
+            show: false,
+          },
+        },
+        yaxis: {
+          labels: {
+            show: false,
+          },
+          logarithmic: true,
+          logBase: 10,
+        },
+      }}
+      series={data}
+      height="100%"
+    />
+  );
+}

web/src/pages/ConfigEditor.tsx

@@ -150,9 +150,9 @@ function ConfigEditor() {
         </div>
       </div>
 
-      {success && <div className="max-h-20 text-green-500">{success}</div>}
+      {success && <div className="max-h-20 text-success">{success}</div>}
       {error && (
-        <div className="p-4 overflow-scroll text-red-500 whitespace-pre-wrap">
+        <div className="p-4 overflow-scroll text-danger whitespace-pre-wrap">
           {error}
         </div>
       )}

web/src/pages/Dashboard.tsx

@@ -123,7 +123,7 @@ function Camera({ camera }: { camera: CameraConfig }) {
                     ? recordValue == "ON"
                       ? "text-primary"
                       : "text-gray-400"
-                    : "text-red-500"
+                    : "text-danger"
                 }
                 onClick={(e) => {
                   e.stopPropagation();

web/src/pages/Export.tsx

@@ -160,7 +160,7 @@ function Export() {
       {message.text && (
         <div
           className={`max-h-20 ${
-            message.error ? "text-red-500" : "text-green-500"
+            message.error ? "text-danger" : "text-success"
           }`}
         >
           {message.text}

web/src/pages/History.tsx

@@ -212,7 +212,7 @@ function History() {
               Cancel
             </AlertDialogCancel>
             <AlertDialogAction
-              className="bg-red-500"
+              className="bg-danger"
               onClick={() => onDeleteMulti()}
             >
               Delete

web/src/types/graph.ts

@@ -0,0 +1,9 @@
+export type GraphDataPoint = {
+  x: Date;
+  y: number;
+};
+
+export type GraphData = {
+  name?: string;
+  data: GraphDataPoint[];
+};

web/src/types/history.ts

@@ -56,6 +56,12 @@ interface HistoryFilter extends FilterType {
   detailLevel: "normal" | "extra" | "full";
 }
 
+type HistoryTimeline = {
+  start: number;
+  end: number;
+  playbackItems: TimelinePlayback[];
+};
+
 type TimelinePlayback = {
   camera: string;
   range: { start: number; end: number };

web/src/types/record.ts

@@ -1,11 +1,30 @@
 type Recording = {
-    id: string,
+  id: string;
-    camera: string,
+  camera: string;
-    start_time: number,
+  start_time: number;
-    end_time: number,
+  end_time: number;
-    path: string,
+  path: string;
-    segment_size: number,
+  segment_size: number;
-    motion: number,
+  motion: number;
-    objects: number,
+  objects: number;
-    dBFS: number,
+  dBFS: number;
-}
+};
+
+type RecordingSegment = {
+  id: string;
+  start_time: number;
+  end_time: number;
+  motion: number;
+  objects: number;
+  segment_size: number;
+};
+
+type RecordingActivity = {
+  [hour: number]: RecordingSegmentActivity[];
+};
+
+type RecordingSegmentActivity = {
+  date: number;
+  count: number;
+  type: "motion" | "objects";
+};

web/src/utils/historyUtil.ts

@@ -107,13 +107,14 @@ export function getTimelineHoursForDay(
   cards: CardsData,
   allPreviews: Preview[],
   timestamp: number
-): TimelinePlayback[] {
+): HistoryTimeline {
   const now = new Date();
   const data: TimelinePlayback[] = [];
   const startDay = new Date(timestamp * 1000);
   startDay.setHours(23, 59, 59, 999);
   const dayEnd = startDay.getTime() / 1000;
   startDay.setHours(0, 0, 0, 0);
+  const startTimestamp = startDay.getTime() / 1000;
   let start = startDay.getTime() / 1000;
   let end = 0;
 
@@ -134,7 +135,7 @@ export function getTimelineHoursForDay(
   });
 
   if (dayIdx == undefined) {
-    return [];
+    return { start: 0, end: 0, playbackItems: [] };
   }
 
   const day = cards[dayIdx];
@@ -179,5 +180,5 @@ export function getTimelineHoursForDay(
     start = startDay.getTime() / 1000;
   }
 
-  return data.reverse();
+  return { start: startTimestamp, end, playbackItems: data.reverse() };
 }

web/src/views/history/DesktopTimelineView.tsx

@@ -10,6 +10,8 @@ import useSWR from "swr";
 import Player from "video.js/dist/types/player";
 import TimelineItemCard from "@/components/card/TimelineItemCard";
 import { getTimelineHoursForDay } from "@/utils/historyUtil";
+import { GraphDataPoint } from "@/types/graph";
+import TimelineGraph from "@/components/graph/TimelineGraph";
 
 type DesktopTimelineViewProps = {
   timelineData: CardsData;
@@ -166,6 +168,50 @@ export default function DesktopTimelineView({
     []
   );
 
+  const { data: activity } = useSWR<RecordingActivity>(
+    [
+      `${initialPlayback.camera}/recording/hourly/activity`,
+      {
+        after: timelineStack.start,
+        before: timelineStack.end,
+        timezone,
+      },
+    ],
+    { revalidateOnFocus: false }
+  );
+  const timelineGraphData = useMemo(() => {
+    if (!activity) {
+      return {};
+    }
+
+    const graphData: {
+      [hour: string]: { objects: GraphDataPoint[]; motion: GraphDataPoint[] };
+    } = {};
+
+    Object.entries(activity).forEach(([hour, data]) => {
+      const objects: GraphDataPoint[] = [];
+      const motion: GraphDataPoint[] = [];
+
+      data.forEach((seg) => {
+        if (seg.type == "objects") {
+          objects.push({
+            x: new Date(seg.date * 1000),
+            y: seg.count,
+          });
+        } else {
+          motion.push({
+            x: new Date(seg.date * 1000),
+            y: seg.count,
+          });
+        }
+      });
+
+      graphData[hour] = { objects, motion };
+    });
+
+    return graphData;
+  }, [activity]);
+
   if (!config) {
     return <ActivityIndicator />;
   }
@@ -271,14 +317,17 @@ export default function DesktopTimelineView({
         </div>
       </div>
       <div className="m-1 max-h-72 2xl:max-h-80 3xl:max-h-96 overflow-auto">
-        {timelineStack.map((timeline) => {
+        {timelineStack.playbackItems.map((timeline) => {
           const isSelected =
             timeline.range.start == selectedPlayback.range.start;
+          const graphData = timelineGraphData[timeline.range.start];
 
           return (
             <div
               key={timeline.range.start}
-              className={`p-2 ${isSelected ? "bg-secondary bg-opacity-30 rounded-md" : ""}`}
+              className={`relative p-2 ${
+                isSelected ? "bg-secondary bg-opacity-30 rounded-md" : ""
+              }`}
             >
               <ActivityScrubber
                 items={[]}
@@ -324,9 +373,6 @@ export default function DesktopTimelineView({
                   setScrubbing(false);
                   playerRef.current?.play();
                 }}
-                doubleClickHandler={() => {
-                  setSelectedPlayback(timeline);
-                }}
                 selectHandler={(data) => {
                   if (data.items.length > 0) {
                     const selected = data.items[0];
@@ -337,7 +383,22 @@ export default function DesktopTimelineView({
                     );
                   }
                 }}
+                doubleClickHandler={() => setSelectedPlayback(timeline)}
               />
+              {isSelected && graphData && (
+                <div className="w-full absolute left-0 top-0 h-[84px]">
+                  <TimelineGraph
+                    id={timeline.range.start.toString()}
+                    data={[
+                      {
+                        name: "Motion",
+                        data: graphData.motion,
+                      },
+                      { name: "Active Objects", data: graphData.objects },
+                    ]}
+                  />
+                </div>
+              )}
             </div>
           );
         })}

web/tailwind.config.js

@@ -20,6 +20,12 @@ module.exports = {
         border: "hsl(var(--border))",
         input: "hsl(var(--input))",
         ring: "hsl(var(--ring))",
+        danger: "#ef4444",
+        success: "#22c55e",
+        // detection colors
+        motion: "#991b1b",
+        object: "#06b6d4",
+        audio: "#ea580c",
         background: "hsl(var(--background))",
         foreground: "hsl(var(--foreground))",
         primary: {