Merge branch 'main' into breaking

common/image_util.py

@@ -0,0 +1,52 @@
+import aiohttp
+import base64
+import io
+import re
+
+from fastapi import HTTPException
+from PIL import Image
+
+from common.networking import (
+    handle_request_error,
+)
+from common.tabby_config import config
+
+
+async def get_image(url: str) -> Image:
+    if url.startswith("data:image"):
+        # Handle base64 image
+        match = re.match(r"^data:image\/[a-zA-Z0-9]+;base64,(.*)$", url)
+        if match:
+            base64_image = match.group(1)
+            bytes_image = base64.b64decode(base64_image)
+        else:
+            error_message = handle_request_error(
+                "Failed to read base64 image input.",
+                exc_info=False,
+            ).error.message
+
+            raise HTTPException(400, error_message)
+
+    else:
+        # Handle image URL
+        if config.network.disable_fetch_requests:
+            error_message = handle_request_error(
+                f"Failed to fetch image from {url} as fetch requests are disabled.",
+                exc_info=False,
+            ).error.message
+
+            raise HTTPException(400, error_message)
+
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url) as response:
+                if response.status == 200:
+                    bytes_image = await response.read()
+                else:
+                    error_message = handle_request_error(
+                        f"Failed to fetch image from {url}.",
+                        exc_info=False,
+                    ).error.message
+
+                    raise HTTPException(400, error_message)
+
+    return Image.open(io.BytesIO(bytes_image))

common/multimodal.py

@@ -1,4 +1,5 @@
-from backends.exllamav2.vision import get_image_embedding
+from backends.exllamav2.vision import get_image_embedding_exl2
+from backends.exllamav3.vision import get_image_embedding_exl3
 from common import model
 from loguru import logger
 from pydantic import BaseModel, Field
@@ -8,6 +9,8 @@ from common.optional_dependencies import dependencies
 
 if dependencies.exllamav2:
     from exllamav2 import ExLlamaV2VisionTower
+if dependencies.exllamav3:
+    from exllamav3 import Model
 
 
 class MultimodalEmbeddingWrapper(BaseModel):
@@ -20,11 +23,22 @@ class MultimodalEmbeddingWrapper(BaseModel):
     async def add(self, url: str):
         # Determine the type of vision embedding to use
         if not self.type:
-            if isinstance(model.container.vision_model, ExLlamaV2VisionTower):
+            if dependencies.exllamav2 and isinstance(
+                model.container.vision_model, ExLlamaV2VisionTower
+            ):
                 self.type = "ExLlamaV2MMEmbedding"
+            elif dependencies.exllamav3 and isinstance(
+                model.container.vision_model, Model
+            ):
+                self.type = "MMEmbedding"
 
+        # Create the embedding
         if self.type == "ExLlamaV2MMEmbedding":
-            embedding = await get_image_embedding(url)
+            embedding = await get_image_embedding_exl2(url)
+            self.content.append(embedding)
+            self.text_alias.append(embedding.text_alias)
+        elif self.type == "MMEmbedding":
+            embedding = await get_image_embedding_exl3(url)
             self.content.append(embedding)
             self.text_alias.append(embedding.text_alias)
         else: