diff --git a/frigate/track/norfair_tracker.py b/frigate/track/norfair_tracker.py
index 04ec21aac..53cdaad6f 100644
--- a/frigate/track/norfair_tracker.py
+++ b/frigate/track/norfair_tracker.py
@@ -17,12 +17,13 @@ from frigate.ptz.autotrack import PtzMotionEstimator
 from frigate.track import ObjectTracker
 from frigate.types import PTZMetricsTypes
 from frigate.util.image import intersection_over_union
-from frigate.util.object import average_boxes
+from frigate.util.object import average_boxes, median_of_boxes
 
 logger = logging.getLogger(__name__)
 
 
-THRESHOLD_STATIONARY_IOU_AVERAGE = 0.6
+THRESHOLD_ACTIVE_IOU = 0.2
+THRESHOLD_STATIONARY_IOU = 0.6
 MAX_STATIONARY_HISTORY = 10
 
 
@@ -146,6 +147,7 @@ class NorfairTracker(ObjectTracker):
     # tracks the current position of the object based on the last N bounding boxes
     # returns False if the object has moved outside its previous position
     def update_position(self, id: str, box: list[int, int, int, int]):
+        xmin, ymin, xmax, ymax = box
         position = self.positions[id]
         self.stationary_box_history[id].append(box)
 
@@ -158,11 +160,9 @@ class NorfairTracker(ObjectTracker):
             box, average_boxes(self.stationary_box_history[id])
         )
 
-        xmin, ymin, xmax, ymax = box
-
-        # if the iou drops below the threshold
-        # assume the object has moved to a new position and reset the computed box
-        if avg_iou < THRESHOLD_STATIONARY_IOU_AVERAGE:
+        # object has minimal or zero iou
+        # assume object is active
+        if avg_iou < THRESHOLD_ACTIVE_IOU:
             self.positions[id] = {
                 "xmins": [xmin],
                 "ymins": [ymin],
@@ -175,6 +175,33 @@ class NorfairTracker(ObjectTracker):
             }
             return False
 
+        # object has iou below threshold, check median to reduce outliers
+        if avg_iou < THRESHOLD_STATIONARY_IOU:
+            median_iou = intersection_over_union(
+                (
+                    position["xmin"],
+                    position["ymin"],
+                    position["xmax"],
+                    position["ymax"],
+                ),
+                median_of_boxes(self.stationary_box_history[id]),
+            )
+
+            # if the median iou drops below the threshold
+            # assume object is no longer stationary
+            if median_iou < THRESHOLD_STATIONARY_IOU:
+                self.positions[id] = {
+                    "xmins": [xmin],
+                    "ymins": [ymin],
+                    "xmaxs": [xmax],
+                    "ymaxs": [ymax],
+                    "xmin": xmin,
+                    "ymin": ymin,
+                    "xmax": xmax,
+                    "ymax": ymax,
+                }
+                return False
+
         # if there are less than 10 entries for the position, add the bounding box
         # and recompute the position box
         if len(position["xmins"]) < 10:
diff --git a/frigate/util/object.py b/frigate/util/object.py
index 81940f303..66810d5e7 100644
--- a/frigate/util/object.py
+++ b/frigate/util/object.py
@@ -339,6 +339,12 @@ def average_boxes(boxes: list[list[int, int, int, int]]) -> list[int, int, int,
     return [np.mean(x_mins), np.mean(y_mins), np.mean(x_max), np.mean(y_max)]
 
 
+def median_of_boxes(boxes: list[list[int, int, int, int]]) -> list[int, int, int, int]:
+    """Return a box that is the median of a list of boxes."""
+    sorted_boxes = sorted(boxes, key=lambda x: area(x))
+    return sorted_boxes[int(len(sorted_boxes) / 2.0)]
+
+
 def intersects_any(box_a, boxes):
     for box in boxes:
         if box_overlaps(box_a, box):