OAI: Add response_format parameter

response_format allows a user to request a valid, but arbitrary JSON
object from the API. This is a new part of the OAI spec.

Signed-off-by: kingbri <bdashore3@proton.me>

endpoints/OAI/router.py

@@ -449,6 +449,10 @@ async def completion_request(request: Request, data: CompletionRequest):
         config.developer_config().get("disable_request_streaming"), False
     )
 
+    # Set an empty JSON schema if the request wants a JSON response
+    if data.response_format.type == "json":
+        data.json_schema = {"type": "object"}
+
     if data.stream and not disable_request_streaming:
         generator_callback = partial(stream_generate_completion, data, model_path)
 
@@ -492,6 +496,10 @@ async def chat_completion_request(request: Request, data: ChatCompletionRequest)
     else:
         prompt = format_prompt_with_template(data)
 
+    # Set an empty JSON schema if the request wants a JSON response
+    if data.response_format.type == "json":
+        data.json_schema = {"type": "object"}
+
     disable_request_streaming = unwrap(
         config.developer_config().get("disable_request_streaming"), False
     )

endpoints/OAI/types/common.py

@@ -14,6 +14,10 @@ class UsageStats(BaseModel):
     total_tokens: int
 
 
+class CompletionResponseFormat(BaseModel):
+    type: str = "text"
+
+
 class CommonCompletionRequest(BaseSamplerRequest):
     """Represents a common completion request."""
 
@@ -24,6 +28,9 @@ class CommonCompletionRequest(BaseSamplerRequest):
     # Generation info (remainder is in BaseSamplerRequest superclass)
     stream: Optional[bool] = False
     logprobs: Optional[int] = 0
+    response_format: Optional[CompletionResponseFormat] = Field(
+        default_factory=CompletionResponseFormat
+    )
 
     # Extra OAI request stuff
     best_of: Optional[int] = Field(