diff --git a/frigate/review/maintainer.py b/frigate/review/maintainer.py
index 3541fef3b..3819f4cb4 100644
--- a/frigate/review/maintainer.py
+++ b/frigate/review/maintainer.py
@@ -253,7 +253,8 @@ class ReviewSegmentMaintainer(threading.Thread):
 
         if len(active_objects) > 0:
             has_activity = True
-            should_update = False
+            should_update_image = False
+            should_update_state = False
 
             if frame_time > segment.last_update:
                 segment.last_update = frame_time
@@ -284,7 +285,8 @@ class ReviewSegmentMaintainer(threading.Thread):
                     and camera_config.review.alerts.enabled
                 ):
                     segment.severity = SeverityEnum.alert
-                    should_update = True
+                    should_update_state = True
+                    should_update_image = True
 
                 # keep zones up to date
                 if len(object["current_zones"]) > 0:
@@ -293,17 +295,24 @@ class ReviewSegmentMaintainer(threading.Thread):
                             segment.zones.append(zone)
 
             if len(active_objects) > segment.frame_active_count:
-                should_update = True
+                should_update_state = True
+                should_update_image = True
 
-            if should_update:
+            if prev_data["data"]["sub_labels"] != list(segment.sub_labels.values()):
+                should_update_state = True
+
+            if should_update_state:
                 try:
-                    yuv_frame = self.frame_manager.get(
-                        frame_name, camera_config.frame_shape_yuv
-                    )
+                    if should_update_image:
+                        yuv_frame = self.frame_manager.get(
+                            frame_name, camera_config.frame_shape_yuv
+                        )
 
-                    if yuv_frame is None:
-                        logger.debug(f"Failed to get frame {frame_name} from SHM")
-                        return
+                        if yuv_frame is None:
+                            logger.debug(f"Failed to get frame {frame_name} from SHM")
+                            return
+                    else:
+                        yuv_frame = None
 
                     self._publish_segment_update(
                         segment, camera_config, yuv_frame, active_objects, prev_data