From 636080261243d7e0103d9b4e40ac332fe4eed4a5 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Wed, 12 Mar 2025 15:38:28 -0500 Subject: [PATCH] Use identifier field for unknown license plates (#17123) * backend * backend fixes * api for search queries * frontend * docs * add filterable scroll list to more filters pane for identifiers * always publish identifier --- .../license_plate_recognition.md | 9 +- docs/docs/integrations/mqtt.md | 8 +- frigate/api/app.py | 33 +++++ .../api/defs/query/events_query_parameters.py | 2 + frigate/api/event.py | 60 ++++++++ frigate/camera/state.py | 9 +- frigate/comms/event_metadata_updater.py | 1 + .../common/license_plate/mixin.py | 12 +- frigate/events/maintainer.py | 6 + frigate/track/object_processing.py | 38 +++++ frigate/track/tracked_object.py | 3 + .../overlay/detail/SearchDetailDialog.tsx | 23 +++ .../overlay/dialog/SearchFilterDialog.tsx | 133 +++++++++++++++++- web/src/pages/Explore.tsx | 2 + web/src/types/search.ts | 4 + web/src/views/search/SearchView.tsx | 4 +- 16 files changed, 332 insertions(+), 15 deletions(-) diff --git a/docs/docs/configuration/license_plate_recognition.md b/docs/docs/configuration/license_plate_recognition.md index 3fe1ee852..f8b9030ff 100644 --- a/docs/docs/configuration/license_plate_recognition.md +++ b/docs/docs/configuration/license_plate_recognition.md @@ -3,16 +3,16 @@ id: license_plate_recognition title: License Plate Recognition (LPR) --- -Frigate can recognize license plates on vehicles and automatically add the detected characters or recognized name as a `sub_label` to objects that are of type `car`. A common use case may be to read the license plates of cars pulling into a driveway or cars passing by on a street. +Frigate can recognize license plates on vehicles and automatically add the detected characters to the `identifier` field or a known name as a `sub_label` to objects that are of type `car`. A common use case may be to read the license plates of cars pulling into a driveway or cars passing by on a street. LPR works best when the license plate is clearly visible to the camera. For moving vehicles, Frigate continuously refines the recognition process, keeping the most confident result. However, LPR does not run on stationary vehicles. -When a plate is recognized, the detected characters or recognized name is: +When a plate is recognized, the recognized name is: -- Added as a `sub_label` to the `car` tracked object. +- Added to the `car` tracked object as a `sub_label` (if known) or the `identifier` field (if unknown) - Viewable in the Review Item Details pane in Review and the Tracked Object Details pane in Explore. - Filterable through the More Filters menu in Explore. -- Published via the `frigate/events` MQTT topic as a `sub_label` for the tracked object. +- Published via the `frigate/events` MQTT topic as a `sub_label` (known) or `identifier` (unknown) for the tracked object. ## Model Requirements @@ -71,6 +71,7 @@ Fine-tune the LPR feature using these optional parameters: - **`known_plates`**: List of strings or regular expressions that assign custom a `sub_label` to `car` objects when a recognized plate matches a known value. - These labels appear in the UI, filters, and notifications. + - Unknown plates are still saved but are added to the `identifier` field rather than the `sub_label`. - **`match_distance`**: Allows for minor variations (missing/incorrect characters) when matching a detected plate to a known plate. - For example, setting `match_distance: 1` allows a plate `ABCDE` to match `ABCBE` or `ABCD`. - This parameter will _not_ operate on known plates that are defined as regular expressions. You should define the full string of your plate in `known_plates` in order to use `match_distance`. diff --git a/docs/docs/integrations/mqtt.md b/docs/docs/integrations/mqtt.md index fc8888e40..bcdbe7046 100644 --- a/docs/docs/integrations/mqtt.md +++ b/docs/docs/integrations/mqtt.md @@ -54,7 +54,9 @@ Message published for each changed tracked object. The first message is publishe }, // attributes with top score that have been identified on the object at any point "current_attributes": [], // detailed data about the current attributes in this frame "current_estimated_speed": 0.71, // current estimated speed (mph or kph) for objects moving through zones with speed estimation enabled - "velocity_angle": 180 // direction of travel relative to the frame for objects moving through zones with speed estimation enabled + "velocity_angle": 180, // direction of travel relative to the frame for objects moving through zones with speed estimation enabled + "identifier": "ABC12345", // an identifier for this object - in this case, an unrecognized license plate + "identifier_score": 0.933451 }, "after": { "id": "1607123955.475377-mxklsc", @@ -93,7 +95,9 @@ Message published for each changed tracked object. The first message is publishe } ], "current_estimated_speed": 0.77, // current estimated speed (mph or kph) for objects moving through zones with speed estimation enabled - "velocity_angle": 180 // direction of travel relative to the frame for objects moving through zones with speed estimation enabled + "velocity_angle": 180, // direction of travel relative to the frame for objects moving through zones with speed estimation enabled + "identifier": "ABC12345", // an identifier for this object - in this case, an unrecognized license plate + "identifier_score": 0.933451 } } ``` diff --git a/frigate/api/app.py b/frigate/api/app.py index 5ce90130f..d9a57a3c1 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -619,6 +619,39 @@ def get_sub_labels(split_joined: Optional[int] = None): return JSONResponse(content=sub_labels) +@router.get("/identifiers") +def get_identifiers(split_joined: Optional[int] = None): + try: + events = Event.select(Event.data).distinct() + except Exception: + return JSONResponse( + content=({"success": False, "message": "Failed to get identifiers"}), + status_code=404, + ) + + identifiers = [] + for e in events: + if e.data is not None and "identifier" in e.data: + identifiers.append(e.data["identifier"]) + + while None in identifiers: + identifiers.remove(None) + + if split_joined: + original_identifiers = identifiers.copy() + for identifier in original_identifiers: + if identifier and "," in identifier: + identifiers.remove(identifier) + parts = identifier.split(",") + for part in parts: + if part.strip() not in identifiers: + identifiers.append(part.strip()) + + identifiers = list(set(identifiers)) + identifiers.sort() + return JSONResponse(content=identifiers) + + @router.get("/timeline") def timeline(camera: str = "all", limit: int = 100, source_id: Optional[str] = None): clauses = [] diff --git a/frigate/api/defs/query/events_query_parameters.py b/frigate/api/defs/query/events_query_parameters.py index 01c79abb0..9f73d8583 100644 --- a/frigate/api/defs/query/events_query_parameters.py +++ b/frigate/api/defs/query/events_query_parameters.py @@ -27,6 +27,7 @@ class EventsQueryParams(BaseModel): max_score: Optional[float] = None min_speed: Optional[float] = None max_speed: Optional[float] = None + identifier: Optional[str] = "all" is_submitted: Optional[int] = None min_length: Optional[float] = None max_length: Optional[float] = None @@ -55,6 +56,7 @@ class EventsSearchQueryParams(BaseModel): max_score: Optional[float] = None min_speed: Optional[float] = None max_speed: Optional[float] = None + identifier: Optional[str] = "all" sort: Optional[str] = None diff --git a/frigate/api/event.py b/frigate/api/event.py index 91651313d..e9cf2fea4 100644 --- a/frigate/api/event.py +++ b/frigate/api/event.py @@ -101,6 +101,7 @@ def events(params: EventsQueryParams = Depends()): min_length = params.min_length max_length = params.max_length event_id = params.event_id + identifier = params.identifier sort = params.sort @@ -158,6 +159,32 @@ def events(params: EventsQueryParams = Depends()): sub_label_clause = reduce(operator.or_, sub_label_clauses) clauses.append((sub_label_clause)) + if identifier != "all": + # use matching so joined identifiers are included + # for example an identifier 'ABC123' would get events + # with identifiers 'ABC123' and 'ABC123, XYZ789' + identifier_clauses = [] + filtered_identifiers = identifier.split(",") + + if "None" in filtered_identifiers: + filtered_identifiers.remove("None") + identifier_clauses.append((Event.data["identifier"].is_null())) + + for identifier in filtered_identifiers: + # Exact matching plus list inclusion + identifier_clauses.append( + (Event.data["identifier"].cast("text") == identifier) + ) + identifier_clauses.append( + (Event.data["identifier"].cast("text") % f"*{identifier},*") + ) + identifier_clauses.append( + (Event.data["identifier"].cast("text") % f"*, {identifier}*") + ) + + identifier_clause = reduce(operator.or_, identifier_clauses) + clauses.append((identifier_clause)) + if zones != "all": # use matching so events with multiple zones # still match on a search where any zone matches @@ -340,6 +367,8 @@ def events_explore(limit: int = 10): "average_estimated_speed", "velocity_angle", "path_data", + "identifier", + "identifier_score", ] }, "event_count": label_counts[event.label], @@ -397,6 +426,7 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends()) has_clip = params.has_clip has_snapshot = params.has_snapshot is_submitted = params.is_submitted + identifier = params.identifier # for similarity search event_id = params.event_id @@ -466,6 +496,32 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends()) event_filters.append((reduce(operator.or_, zone_clauses))) + if identifier != "all": + # use matching so joined identifiers are included + # for example an identifier 'ABC123' would get events + # with identifiers 'ABC123' and 'ABC123, XYZ789' + identifier_clauses = [] + filtered_identifiers = identifier.split(",") + + if "None" in filtered_identifiers: + filtered_identifiers.remove("None") + identifier_clauses.append((Event.data["identifier"].is_null())) + + for identifier in filtered_identifiers: + # Exact matching plus list inclusion + identifier_clauses.append( + (Event.data["identifier"].cast("text") == identifier) + ) + identifier_clauses.append( + (Event.data["identifier"].cast("text") % f"*{identifier},*") + ) + identifier_clauses.append( + (Event.data["identifier"].cast("text") % f"*, {identifier}*") + ) + + identifier_clause = reduce(operator.or_, identifier_clauses) + event_filters.append((identifier_clause)) + if after: event_filters.append((Event.start_time > after)) @@ -627,6 +683,8 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends()) "average_estimated_speed", "velocity_angle", "path_data", + "identifier", + "identifier_score", ] } @@ -681,6 +739,7 @@ def events_summary(params: EventsSummaryQueryParams = Depends()): Event.camera, Event.label, Event.sub_label, + Event.data, fn.strftime( "%Y-%m-%d", fn.datetime( @@ -695,6 +754,7 @@ def events_summary(params: EventsSummaryQueryParams = Depends()): Event.camera, Event.label, Event.sub_label, + Event.data, (Event.start_time + seconds_offset).cast("int") / (3600 * 24), Event.zones, ) diff --git a/frigate/camera/state.py b/frigate/camera/state.py index dfd6744e2..2ffa54798 100644 --- a/frigate/camera/state.py +++ b/frigate/camera/state.py @@ -137,12 +137,13 @@ class CameraState: # draw the bounding boxes on the frame box = obj["box"] text = ( - obj["label"] + obj["sub_label"][0] if ( - not obj.get("sub_label") - or not is_label_printable(obj["sub_label"][0]) + obj.get("sub_label") and is_label_printable(obj["sub_label"][0]) ) - else obj["sub_label"][0] + else obj.get("identifier", [None])[0] + if (obj.get("identifier") and obj["identifier"][0]) + else obj["label"] ) draw_box_with_label( frame_copy, diff --git a/frigate/comms/event_metadata_updater.py b/frigate/comms/event_metadata_updater.py index 3342182c3..c5881e686 100644 --- a/frigate/comms/event_metadata_updater.py +++ b/frigate/comms/event_metadata_updater.py @@ -14,6 +14,7 @@ class EventMetadataTypeEnum(str, Enum): manual_event_end = "manual_event_end" regenerate_description = "regenerate_description" sub_label = "sub_label" + identifier = "identifier" class EventMetadataPublisher(Publisher): diff --git a/frigate/data_processing/common/license_plate/mixin.py b/frigate/data_processing/common/license_plate/mixin.py index c74949d9c..37530b205 100644 --- a/frigate/data_processing/common/license_plate/mixin.py +++ b/frigate/data_processing/common/license_plate/mixin.py @@ -1054,13 +1054,19 @@ class LicensePlateProcessingMixin: for plate in plates ) ), - top_plate, + None, ) - # Send the result to the API + # If it's a known plate, publish to sub_label + if sub_label is not None: + self.sub_label_publisher.publish( + EventMetadataTypeEnum.sub_label, (id, sub_label, avg_confidence) + ) + self.sub_label_publisher.publish( - EventMetadataTypeEnum.sub_label, (id, sub_label, avg_confidence) + EventMetadataTypeEnum.identifier, (id, top_plate, avg_confidence) ) + self.detected_license_plates[id] = { "plate": top_plate, "char_confidences": top_char_confidences, diff --git a/frigate/events/maintainer.py b/frigate/events/maintainer.py index 5cfa7c716..947763f40 100644 --- a/frigate/events/maintainer.py +++ b/frigate/events/maintainer.py @@ -27,6 +27,7 @@ def should_update_db(prev_event: Event, current_event: Event) -> bool: or prev_event["average_estimated_speed"] != current_event["average_estimated_speed"] or prev_event["velocity_angle"] != current_event["velocity_angle"] + or prev_event["identifier"] != current_event["identifier"] or prev_event["path_data"] != current_event["path_data"] ): return True @@ -226,6 +227,11 @@ class EventProcessor(threading.Thread): event[Event.sub_label] = event_data["sub_label"][0] event[Event.data]["sub_label_score"] = event_data["sub_label"][1] + # only overwrite the identifier in the database if it's set + if event_data.get("identifier") is not None: + event[Event.data]["identifier"] = event_data["identifier"][0] + event[Event.data]["identifier_score"] = event_data["identifier"][1] + ( Event.insert(event) .on_conflict( diff --git a/frigate/track/object_processing.py b/frigate/track/object_processing.py index ddac2b588..56dd59110 100644 --- a/frigate/track/object_processing.py +++ b/frigate/track/object_processing.py @@ -346,6 +346,41 @@ class TrackedObjectProcessor(threading.Thread): return True + def set_identifier( + self, event_id: str, identifier: str | None, score: float | None + ) -> None: + """Update identifier for given event id.""" + tracked_obj: TrackedObject = None + + for state in self.camera_states.values(): + tracked_obj = state.tracked_objects.get(event_id) + + if tracked_obj is not None: + break + + try: + event: Event = Event.get(Event.id == event_id) + except DoesNotExist: + event = None + + if not tracked_obj and not event: + return + + if tracked_obj: + tracked_obj.obj_data["identifier"] = (identifier, score) + + if event: + data = event.data + data["identifier"] = identifier + if identifier is None: + data["identifier_score"] = None + elif score is not None: + data["identifier_score"] = score + event.data = data + event.save() + + return True + def create_manual_event(self, payload: tuple) -> None: ( frame_time, @@ -507,6 +542,9 @@ class TrackedObjectProcessor(threading.Thread): if topic.endswith(EventMetadataTypeEnum.sub_label.value): (event_id, sub_label, score) = payload self.set_sub_label(event_id, sub_label, score) + if topic.endswith(EventMetadataTypeEnum.identifier.value): + (event_id, identifier, score) = payload + self.set_identifier(event_id, identifier, score) elif topic.endswith(EventMetadataTypeEnum.manual_event_create.value): self.create_manual_event(payload) elif topic.endswith(EventMetadataTypeEnum.manual_event_end.value): diff --git a/frigate/track/tracked_object.py b/frigate/track/tracked_object.py index f1eb29328..9c19595b5 100644 --- a/frigate/track/tracked_object.py +++ b/frigate/track/tracked_object.py @@ -153,6 +153,8 @@ class TrackedObject: "current_estimated_speed": self.current_estimated_speed, "velocity_angle": self.velocity_angle, "path_data": self.path_data, + "identifier": obj_data.get("identifier"), + "identifier_score": obj_data.get("identifier_score"), } thumb_update = True @@ -365,6 +367,7 @@ class TrackedObject: "average_estimated_speed": self.average_estimated_speed, "velocity_angle": self.velocity_angle, "path_data": self.path_data, + "identifier": self.obj_data.get("identifier"), } return event diff --git a/web/src/components/overlay/detail/SearchDetailDialog.tsx b/web/src/components/overlay/detail/SearchDetailDialog.tsx index c94c2cd2d..7f69f2ca2 100644 --- a/web/src/components/overlay/detail/SearchDetailDialog.tsx +++ b/web/src/components/overlay/detail/SearchDetailDialog.tsx @@ -333,6 +333,18 @@ function ObjectDetailsTab({ } }, [search]); + const identifierScore = useMemo(() => { + if (!search) { + return undefined; + } + + if (search.data.identifier && search.data?.identifier_score) { + return Math.round((search.data?.identifier_score ?? 0) * 100); + } else { + return undefined; + } + }, [search]); + const averageEstimatedSpeed = useMemo(() => { if (!search || !search.data?.average_estimated_speed) { return undefined; @@ -538,6 +550,17 @@ function ObjectDetailsTab({ + {search?.data.identifier && ( +
+
Identifier
+
+
+ {search.data.identifier}{" "} + {identifierScore && ` (${identifierScore}%)`} +
+
+
+ )}
diff --git a/web/src/components/overlay/dialog/SearchFilterDialog.tsx b/web/src/components/overlay/dialog/SearchFilterDialog.tsx index 23deee531..59b30f82f 100644 --- a/web/src/components/overlay/dialog/SearchFilterDialog.tsx +++ b/web/src/components/overlay/dialog/SearchFilterDialog.tsx @@ -33,6 +33,14 @@ import { TooltipProvider, TooltipTrigger, } from "@/components/ui/tooltip"; +import { + Command, + CommandEmpty, + CommandInput, + CommandItem, + CommandList, +} from "@/components/ui/command"; +import { LuCheck } from "react-icons/lu"; type SearchFilterDialogProps = { config?: FrigateConfig; @@ -77,7 +85,8 @@ export default function SearchFilterDialog({ (currentFilter.max_score ?? 1) < 1 || (currentFilter.max_speed ?? 150) < 150 || (currentFilter.zones?.length ?? 0) > 0 || - (currentFilter.sub_labels?.length ?? 0) > 0), + (currentFilter.sub_labels?.length ?? 0) > 0 || + (currentFilter.identifier?.length ?? 0) > 0), [currentFilter], ); @@ -119,6 +128,12 @@ export default function SearchFilterDialog({ setCurrentFilter({ ...currentFilter, sub_labels: newSubLabels }) } /> + + setCurrentFilter({ ...currentFilter, identifier: identifiers }) + } + /> @@ -830,3 +846,118 @@ export function SnapshotClipFilterContent({
); } + +type IdentifierFilterContentProps = { + identifiers: string[] | undefined; + setIdentifiers: (identifiers: string[] | undefined) => void; +}; + +export function IdentifierFilterContent({ + identifiers, + setIdentifiers, +}: IdentifierFilterContentProps) { + const { data: allIdentifiers, error } = useSWR("identifiers", { + revalidateOnFocus: false, + }); + + const [selectedIdentifiers, setSelectedIdentifiers] = useState( + identifiers || [], + ); + const [inputValue, setInputValue] = useState(""); + + useEffect(() => { + if (identifiers) { + setSelectedIdentifiers(identifiers); + } else { + setSelectedIdentifiers([]); + } + }, [identifiers]); + + const handleSelect = (identifier: string) => { + const newSelected = selectedIdentifiers.includes(identifier) + ? selectedIdentifiers.filter((id) => id !== identifier) // Deselect + : [...selectedIdentifiers, identifier]; // Select + + setSelectedIdentifiers(newSelected); + if (newSelected.length === 0) { + setIdentifiers(undefined); // Clear filter if no identifiers selected + } else { + setIdentifiers(newSelected); + } + }; + + if (!allIdentifiers || allIdentifiers.length === 0) { + return null; + } + + const filteredIdentifiers = + allIdentifiers?.filter((id) => + id.toLowerCase().includes(inputValue.toLowerCase()), + ) || []; + + return ( +
+ +
Identifiers
+ {error ? ( +

Failed to load identifiers

+ ) : !allIdentifiers ? ( +

Loading identifiers...

+ ) : ( + <> + + + + {filteredIdentifiers.length === 0 && inputValue && ( + No identifiers found. + )} + {filteredIdentifiers.map((identifier) => ( + handleSelect(identifier)} + className="cursor-pointer" + > + + {identifier} + + ))} + + + {selectedIdentifiers.length > 0 && ( +
+ {selectedIdentifiers.map((id) => ( + + {id} + + + ))} +
+ )} + + )} +

+ Select one or more identifiers from the list. +

+
+ ); +} diff --git a/web/src/pages/Explore.tsx b/web/src/pages/Explore.tsx index af23c18f4..cf24ff70d 100644 --- a/web/src/pages/Explore.tsx +++ b/web/src/pages/Explore.tsx @@ -105,6 +105,7 @@ export default function Explore() { cameras: searchSearchParams["cameras"], labels: searchSearchParams["labels"], sub_labels: searchSearchParams["sub_labels"], + identifier: searchSearchParams["identifier"], zones: searchSearchParams["zones"], before: searchSearchParams["before"], after: searchSearchParams["after"], @@ -140,6 +141,7 @@ export default function Explore() { cameras: searchSearchParams["cameras"], labels: searchSearchParams["labels"], sub_labels: searchSearchParams["sub_labels"], + identifier: searchSearchParams["identifier"], zones: searchSearchParams["zones"], before: searchSearchParams["before"], after: searchSearchParams["after"], diff --git a/web/src/types/search.ts b/web/src/types/search.ts index cef7f6aff..90bcd54d7 100644 --- a/web/src/types/search.ts +++ b/web/src/types/search.ts @@ -58,6 +58,8 @@ export type SearchResult = { average_estimated_speed: number; velocity_angle: number; path_data: [number[], number][]; + identifier?: string; + identifier_score?: number; }; }; @@ -66,6 +68,7 @@ export type SearchFilter = { cameras?: string[]; labels?: string[]; sub_labels?: string[]; + identifier?: string[]; zones?: string[]; before?: number; after?: number; @@ -89,6 +92,7 @@ export type SearchQueryParams = { cameras?: string[]; labels?: string[]; sub_labels?: string[]; + identifier?: string[]; zones?: string[]; before?: string; after?: string; diff --git a/web/src/views/search/SearchView.tsx b/web/src/views/search/SearchView.tsx index adbc96413..7f1b1e4a1 100644 --- a/web/src/views/search/SearchView.tsx +++ b/web/src/views/search/SearchView.tsx @@ -121,6 +121,7 @@ export default function SearchView({ }, [config, searchFilter]); const { data: allSubLabels } = useSWR("sub_labels"); + const { data: allIdentifiers } = useSWR("identifiers"); const allZones = useMemo(() => { if (!config) { @@ -160,12 +161,13 @@ export default function SearchView({ max_score: ["100"], min_speed: ["1"], max_speed: ["150"], + identifier: allIdentifiers, has_clip: ["yes", "no"], has_snapshot: ["yes", "no"], ...(config?.plus?.enabled && searchFilter?.has_snapshot && { is_submitted: ["yes", "no"] }), }), - [config, allLabels, allZones, allSubLabels, searchFilter], + [config, allLabels, allZones, allSubLabels, allIdentifiers, searchFilter], ); // remove duplicate event ids