Mirrors/blakeblackshear.frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2026-02-20 13:54:36 +01:00
Code Issues Projects Releases Wiki Activity

Add ability to use Jina CLIP V2 for semantic search (#16826)

Browse Source 
* add wheels

* move extra index url to bottom

* config model option

* add postprocess

* fix config

* jina v2 embedding class

* use jina v2 in embeddings

* fix ov inference

* frontend

* update reference config

* revert device

* fix truncation

* return np tensors

* use correct embeddings from inference

* manual preprocess

* clean up

* docs

* lower batch size for v2 only

* docs clarity

* wording
This commit is contained in:
Josh Hawkins
2025-02-26 08:58:25 -06:00
committed by GitHub
parent 447f26e1b9
commit d0e9bcbfdc
10 changed files with 380 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
web/src/pages/Explore.tsx
View File

@@ -267,20 +267,41 @@ export default function Explore() {
// model states
const { payload: textModelState } = useModelState(
"jinaai/jina-clip-v1-text_model_fp16.onnx",
);
const { payload: textTokenizerState } = useModelState(
"jinaai/jina-clip-v1-tokenizer",
);
const modelFile =
config?.semantic_search.model_size === "large"
? "jinaai/jina-clip-v1-vision_model_fp16.onnx"
: "jinaai/jina-clip-v1-vision_model_quantized.onnx";
const modelVersion = config?.semantic_search.model || "jinav1";
const modelSize = config?.semantic_search.model_size || "small";
const { payload: visionModelState } = useModelState(modelFile);
// Text model state
const { payload: textModelState } = useModelState(
modelVersion === "jinav1"
? "jinaai/jina-clip-v1-text_model_fp16.onnx"
: modelSize === "large"
? "jinaai/jina-clip-v2-model_fp16.onnx"
: "jinaai/jina-clip-v2-model_quantized.onnx",
);
// Tokenizer state
const { payload: textTokenizerState } = useModelState(
modelVersion === "jinav1"
? "jinaai/jina-clip-v1-tokenizer"
: "jinaai/jina-clip-v2-tokenizer",
);
// Vision model state (same as text model for jinav2)
const visionModelFile =
modelVersion === "jinav1"
? modelSize === "large"
? "jinaai/jina-clip-v1-vision_model_fp16.onnx"
: "jinaai/jina-clip-v1-vision_model_quantized.onnx"
: modelSize === "large"
? "jinaai/jina-clip-v2-model_fp16.onnx"
: "jinaai/jina-clip-v2-model_quantized.onnx";
const { payload: visionModelState } = useModelState(visionModelFile);
// Preprocessor/feature extractor state
const { payload: visionFeatureExtractorState } = useModelState(
"jinaai/jina-clip-v1-preprocessor_config.json",
modelVersion === "jinav1"
? "jinaai/jina-clip-v1-preprocessor_config.json"
: "jinaai/jina-clip-v2-preprocessor_config.json",
);
const allModelsLoaded = useMemo(() => {

2
web/src/types/frigateConfig.ts
View File

@@ -20,6 +20,7 @@ export interface BirdseyeConfig {
width: number;
}
export type SearchModel = "jinav1" | "jinav2";
export type SearchModelSize = "small" | "large";
export interface CameraConfig {
@@ -458,6 +459,7 @@ export interface FrigateConfig {
semantic_search: {
enabled: boolean;
reindex: boolean;
model: SearchModel;
model_size: SearchModelSize;
};