jalr/tabbyAPI-ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Exl3: Add vision capability

Browse source
This commit is contained in:
turboderp 2025-06-15 19:22:51 +02:00
parent 4605c0f6bd
commit 1c9891bf04
4 changed files with 80 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

25
common/multimodal.py
View file

@ -1,4 +1,5 @@
from backends.exllamav2.vision import get_image_embedding
from backends.exllamav2.vision import get_image_embedding_exl2
from backends.exllamav3.vision import get_image_embedding_exl3
from common import model
from loguru import logger
from pydantic import BaseModel, Field
@ -8,7 +9,8 @@ from common.optional_dependencies import dependencies
if dependencies.exllamav2:
from exllamav2 import ExLlamaV2VisionTower
if dependencies.exllamav3:
from exllamav3 import Model
class MultimodalEmbeddingWrapper(BaseModel):
"""Common multimodal embedding wrapper"""
@ -20,12 +22,25 @@ class MultimodalEmbeddingWrapper(BaseModel):
async def add(self, url: str):
# Determine the type of vision embedding to use
if not self.type:
if isinstance(model.container.vision_model, ExLlamaV2VisionTower):
if (
dependencies.exllamav2 and
isinstance(model.container.vision_model, ExLlamaV2VisionTower)
):
self.type = "ExLlamaV2MMEmbedding"
elif (
dependencies.exllamav3 and
isinstance(model.container.vision_model, Model)
):
self.type = "MMEmbedding"
# Create the embedding
if self.type == "ExLlamaV2MMEmbedding":
embedding = await get_image_embedding(url)
embedding = await get_image_embedding_exl2(url)
self.content.append(embedding)
self.text_alias.append(embedding.text_alias)
elif self.type == "MMEmbedding":
embedding = await get_image_embedding_exl3(url)
self.content.append(embedding)
self.text_alias.append(embedding.text_alias)
else:
logger.error("No valid vision model to create embedding")
logger.error("No valid vision model to create embedding")