chore: initial public snapshot for github upload
This commit is contained in:
Your Name
@@ -0,0 +1,161 @@
|
||||
# JSON-Based OpenAI-Compatible Provider Configuration
|
||||
|
||||
This directory contains the new JSON-based configuration system for OpenAI-compatible providers.
|
||||
|
||||
## Overview
|
||||
|
||||
Instead of creating a full Python module for simple OpenAI-compatible providers, you can now define them in a single JSON file.
|
||||
|
||||
## Files
|
||||
|
||||
- `providers.json` - Configuration file for all JSON-based providers
|
||||
- `json_loader.py` - Loads and parses the JSON configuration
|
||||
- `dynamic_config.py` - Generates Python config classes from JSON (chat + responses)
|
||||
- `chat/` - OpenAI-like chat completion handlers
|
||||
- `responses/` - OpenAI-like Responses API handlers
|
||||
|
||||
## Adding a New Provider
|
||||
|
||||
### For Simple OpenAI-Compatible Providers
|
||||
|
||||
Edit `providers.json` and add your provider:
|
||||
|
||||
```json
|
||||
{
|
||||
"your_provider": {
|
||||
"base_url": "https://api.yourprovider.com/v1",
|
||||
"api_key_env": "YOUR_PROVIDER_API_KEY"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
That's it! The provider will be automatically loaded and available.
|
||||
|
||||
### Optional Configuration Fields
|
||||
|
||||
```json
|
||||
{
|
||||
"your_provider": {
|
||||
"base_url": "https://api.yourprovider.com/v1",
|
||||
"api_key_env": "YOUR_PROVIDER_API_KEY",
|
||||
|
||||
// Optional: Override base_url via environment variable
|
||||
"api_base_env": "YOUR_PROVIDER_API_BASE",
|
||||
|
||||
// Optional: Which base class to use (default: "openai_gpt")
|
||||
"base_class": "openai_gpt", // or "openai_like"
|
||||
|
||||
// Optional: Parameter name mappings
|
||||
"param_mappings": {
|
||||
"max_completion_tokens": "max_tokens"
|
||||
},
|
||||
|
||||
// Optional: Parameter constraints
|
||||
"constraints": {
|
||||
"temperature_max": 1.0,
|
||||
"temperature_min": 0.0,
|
||||
"temperature_min_with_n_gt_1": 0.3
|
||||
},
|
||||
|
||||
// Optional: Special handling flags
|
||||
"special_handling": {
|
||||
"convert_content_list_to_string": true
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Example: PublicAI
|
||||
|
||||
The first JSON-configured provider:
|
||||
|
||||
```json
|
||||
{
|
||||
"publicai": {
|
||||
"base_url": "https://api.publicai.co/v1",
|
||||
"api_key_env": "PUBLICAI_API_KEY",
|
||||
"api_base_env": "PUBLICAI_API_BASE",
|
||||
"base_class": "openai_gpt",
|
||||
"param_mappings": {
|
||||
"max_completion_tokens": "max_tokens"
|
||||
},
|
||||
"special_handling": {
|
||||
"convert_content_list_to_string": true
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
model="publicai/swiss-ai/apertus-8b-instruct",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
)
|
||||
```
|
||||
|
||||
## Responses API Support
|
||||
|
||||
Providers that support the OpenAI Responses API (`/v1/responses`) can declare it via `supported_endpoints`:
|
||||
|
||||
```json
|
||||
{
|
||||
"your_provider": {
|
||||
"base_url": "https://api.yourprovider.com/v1",
|
||||
"api_key_env": "YOUR_PROVIDER_API_KEY",
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
This enables `litellm.responses(model="your_provider/model-name", ...)` with zero Python code.
|
||||
The provider inherits all request/response handling from OpenAI's Responses API config.
|
||||
|
||||
If `supported_endpoints` is omitted, it defaults to `[]` (only chat completions, which is always enabled for JSON providers).
|
||||
|
||||
### How It Works
|
||||
|
||||
1. `json_loader.py` checks `supported_endpoints` for `/v1/responses`
|
||||
2. `dynamic_config.py` generates a responses config class (inherits from `OpenAIResponsesAPIConfig`)
|
||||
3. `ProviderConfigManager.get_provider_responses_api_config()` returns the generated config
|
||||
4. Request/response transformation is inherited from OpenAI — no custom code needed
|
||||
|
||||
## Benefits
|
||||
|
||||
- **Simple**: 2-5 lines of JSON vs 100+ lines of Python
|
||||
- **Fast**: Add a provider in 5 minutes
|
||||
- **Safe**: No Python code to mess up
|
||||
- **Consistent**: All providers follow the same pattern
|
||||
- **Maintainable**: Centralized configuration
|
||||
|
||||
## When to Use Python Instead
|
||||
|
||||
Use a Python config class if you need:
|
||||
- Custom authentication (OAuth, rotating tokens, etc.)
|
||||
- Complex request/response transformations
|
||||
- Provider-specific streaming logic
|
||||
- Advanced tool calling transformations
|
||||
|
||||
For providers that are *mostly* OpenAI-compatible but need small overrides (e.g. preset model handling),
|
||||
you can inherit from `OpenAIResponsesAPIConfig` and override only what's needed — see
|
||||
`litellm/llms/perplexity/responses/transformation.py` for a minimal example (~40 lines).
|
||||
|
||||
## Implementation Details
|
||||
|
||||
### How It Works
|
||||
|
||||
1. `json_loader.py` loads `providers.json` on import
|
||||
2. `dynamic_config.py` generates config classes on-demand
|
||||
3. Provider resolution checks JSON registry first
|
||||
4. ProviderConfigManager returns JSON-based configs
|
||||
|
||||
### Integration Points
|
||||
|
||||
The JSON system is integrated at:
|
||||
- `litellm/litellm_core_utils/get_llm_provider_logic.py` - Provider resolution
|
||||
- `litellm/utils.py` - ProviderConfigManager (chat + responses)
|
||||
- `litellm/responses/main.py` - Responses API routing
|
||||
- `litellm/constants.py` - openai_compatible_providers list
|
||||
@@ -0,0 +1,403 @@
|
||||
"""
|
||||
OpenAI-like chat completion handler
|
||||
|
||||
For handling OpenAI-like chat completions, like IBM WatsonX, etc.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Any, Callable, Optional, Union
|
||||
|
||||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm import LlmProviders
|
||||
from litellm.llms.bedrock.chat.invoke_handler import MockResponseIterator
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.llms.databricks.streaming_utils import ModelResponseIterator
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from litellm.llms.openai.openai import OpenAIConfig
|
||||
from litellm.types.utils import CustomStreamingDecoder, ModelResponse
|
||||
from litellm.utils import CustomStreamWrapper, ProviderConfigManager
|
||||
|
||||
from ..common_utils import OpenAILikeBase, OpenAILikeError
|
||||
from .transformation import OpenAILikeChatConfig
|
||||
|
||||
|
||||
async def make_call(
|
||||
client: Optional[AsyncHTTPHandler],
|
||||
api_base: str,
|
||||
headers: dict,
|
||||
data: str,
|
||||
model: str,
|
||||
messages: list,
|
||||
logging_obj,
|
||||
streaming_decoder: Optional[CustomStreamingDecoder] = None,
|
||||
fake_stream: bool = False,
|
||||
):
|
||||
if client is None:
|
||||
client = litellm.module_level_aclient
|
||||
|
||||
response = await client.post(
|
||||
api_base, headers=headers, data=data, stream=not fake_stream
|
||||
)
|
||||
|
||||
if streaming_decoder is not None:
|
||||
completion_stream: Any = streaming_decoder.aiter_bytes(
|
||||
response.aiter_bytes(chunk_size=1024)
|
||||
)
|
||||
elif fake_stream:
|
||||
model_response = ModelResponse(**response.json())
|
||||
completion_stream = MockResponseIterator(model_response=model_response)
|
||||
else:
|
||||
completion_stream = ModelResponseIterator(
|
||||
streaming_response=response.aiter_lines(), sync_stream=False
|
||||
)
|
||||
# LOGGING
|
||||
logging_obj.post_call(
|
||||
input=messages,
|
||||
api_key="",
|
||||
original_response=completion_stream, # Pass the completion stream for logging
|
||||
additional_args={"complete_input_dict": data},
|
||||
)
|
||||
|
||||
return completion_stream
|
||||
|
||||
|
||||
def make_sync_call(
|
||||
client: Optional[HTTPHandler],
|
||||
api_base: str,
|
||||
headers: dict,
|
||||
data: str,
|
||||
model: str,
|
||||
messages: list,
|
||||
logging_obj,
|
||||
streaming_decoder: Optional[CustomStreamingDecoder] = None,
|
||||
fake_stream: bool = False,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
):
|
||||
if client is None:
|
||||
client = litellm.module_level_client # Create a new client if none provided
|
||||
|
||||
response = client.post(
|
||||
api_base, headers=headers, data=data, stream=not fake_stream, timeout=timeout
|
||||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise OpenAILikeError(status_code=response.status_code, message=response.read())
|
||||
|
||||
if streaming_decoder is not None:
|
||||
completion_stream = streaming_decoder.iter_bytes(
|
||||
response.iter_bytes(chunk_size=1024)
|
||||
)
|
||||
elif fake_stream:
|
||||
model_response = ModelResponse(**response.json())
|
||||
completion_stream = MockResponseIterator(model_response=model_response)
|
||||
else:
|
||||
completion_stream = ModelResponseIterator(
|
||||
streaming_response=response.iter_lines(), sync_stream=True
|
||||
)
|
||||
|
||||
# LOGGING
|
||||
logging_obj.post_call(
|
||||
input=messages,
|
||||
api_key="",
|
||||
original_response="first stream response received",
|
||||
additional_args={"complete_input_dict": data},
|
||||
)
|
||||
|
||||
return completion_stream
|
||||
|
||||
|
||||
class OpenAILikeChatHandler(OpenAILikeBase):
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
async def acompletion_stream_function(
|
||||
self,
|
||||
model: str,
|
||||
messages: list,
|
||||
custom_llm_provider: str,
|
||||
api_base: str,
|
||||
custom_prompt_dict: dict,
|
||||
model_response: ModelResponse,
|
||||
print_verbose: Callable,
|
||||
encoding,
|
||||
api_key,
|
||||
logging_obj,
|
||||
stream,
|
||||
data: dict,
|
||||
optional_params=None,
|
||||
litellm_params=None,
|
||||
logger_fn=None,
|
||||
headers={},
|
||||
client: Optional[AsyncHTTPHandler] = None,
|
||||
streaming_decoder: Optional[CustomStreamingDecoder] = None,
|
||||
fake_stream: bool = False,
|
||||
) -> CustomStreamWrapper:
|
||||
data["stream"] = True
|
||||
completion_stream = await make_call(
|
||||
client=client,
|
||||
api_base=api_base,
|
||||
headers=headers,
|
||||
data=json.dumps(data),
|
||||
model=model,
|
||||
messages=messages,
|
||||
logging_obj=logging_obj,
|
||||
streaming_decoder=streaming_decoder,
|
||||
)
|
||||
streamwrapper = CustomStreamWrapper(
|
||||
completion_stream=completion_stream,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
return streamwrapper
|
||||
|
||||
async def acompletion_function(
|
||||
self,
|
||||
model: str,
|
||||
messages: list,
|
||||
api_base: str,
|
||||
custom_prompt_dict: dict,
|
||||
model_response: ModelResponse,
|
||||
custom_llm_provider: str,
|
||||
print_verbose: Callable,
|
||||
client: Optional[AsyncHTTPHandler],
|
||||
encoding,
|
||||
api_key,
|
||||
logging_obj,
|
||||
stream,
|
||||
data: dict,
|
||||
base_model: Optional[str],
|
||||
optional_params: dict,
|
||||
litellm_params=None,
|
||||
logger_fn=None,
|
||||
headers={},
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
json_mode: bool = False,
|
||||
) -> ModelResponse:
|
||||
if timeout is None:
|
||||
timeout = httpx.Timeout(timeout=600.0, connect=5.0)
|
||||
|
||||
if client is None:
|
||||
client = litellm.module_level_aclient
|
||||
|
||||
try:
|
||||
response = await client.post(
|
||||
api_base, headers=headers, data=json.dumps(data), timeout=timeout
|
||||
)
|
||||
response.raise_for_status()
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise OpenAILikeError(
|
||||
status_code=e.response.status_code,
|
||||
message=e.response.text,
|
||||
)
|
||||
except httpx.TimeoutException:
|
||||
raise OpenAILikeError(status_code=408, message="Timeout error occurred.")
|
||||
except Exception as e:
|
||||
raise OpenAILikeError(status_code=500, message=str(e))
|
||||
|
||||
return OpenAILikeChatConfig._transform_response(
|
||||
model=model,
|
||||
response=response,
|
||||
model_response=model_response,
|
||||
stream=stream,
|
||||
logging_obj=logging_obj,
|
||||
optional_params=optional_params,
|
||||
api_key=api_key,
|
||||
data=data,
|
||||
messages=messages,
|
||||
print_verbose=print_verbose,
|
||||
encoding=encoding,
|
||||
json_mode=json_mode,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
base_model=base_model,
|
||||
)
|
||||
|
||||
def completion(
|
||||
self,
|
||||
*,
|
||||
model: str,
|
||||
messages: list,
|
||||
api_base: str,
|
||||
custom_llm_provider: str,
|
||||
custom_prompt_dict: dict,
|
||||
model_response: ModelResponse,
|
||||
print_verbose: Callable,
|
||||
encoding,
|
||||
api_key: Optional[str],
|
||||
logging_obj,
|
||||
optional_params: dict,
|
||||
acompletion=None,
|
||||
litellm_params: dict = {},
|
||||
logger_fn=None,
|
||||
headers: Optional[dict] = None,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
|
||||
custom_endpoint: Optional[bool] = None,
|
||||
streaming_decoder: Optional[
|
||||
CustomStreamingDecoder
|
||||
] = None, # if openai-compatible api needs custom stream decoder - e.g. sagemaker
|
||||
fake_stream: bool = False,
|
||||
):
|
||||
custom_endpoint = custom_endpoint or optional_params.pop(
|
||||
"custom_endpoint", None
|
||||
)
|
||||
base_model: Optional[str] = optional_params.pop("base_model", None)
|
||||
api_base, headers = self._validate_environment(
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
endpoint_type="chat_completions",
|
||||
custom_endpoint=custom_endpoint,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
stream: bool = optional_params.pop("stream", None) or False
|
||||
extra_body = optional_params.pop("extra_body", {})
|
||||
json_mode = optional_params.pop("json_mode", None)
|
||||
optional_params.pop("max_retries", None)
|
||||
if not fake_stream:
|
||||
optional_params["stream"] = stream
|
||||
|
||||
if messages is not None and custom_llm_provider is not None:
|
||||
provider_config = ProviderConfigManager.get_provider_chat_config(
|
||||
model=model, provider=LlmProviders(custom_llm_provider)
|
||||
)
|
||||
if isinstance(provider_config, OpenAIGPTConfig) or isinstance(
|
||||
provider_config, OpenAIConfig
|
||||
):
|
||||
messages = provider_config._transform_messages(
|
||||
messages=messages, model=model
|
||||
)
|
||||
|
||||
data = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
**optional_params,
|
||||
**extra_body,
|
||||
}
|
||||
|
||||
## LOGGING
|
||||
logging_obj.pre_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
additional_args={
|
||||
"complete_input_dict": data,
|
||||
"api_base": api_base,
|
||||
"headers": headers,
|
||||
},
|
||||
)
|
||||
if acompletion is True:
|
||||
if client is None or not isinstance(client, AsyncHTTPHandler):
|
||||
client = None
|
||||
if (
|
||||
stream is True
|
||||
): # if function call - fake the streaming (need complete blocks for output parsing in openai format)
|
||||
data["stream"] = stream
|
||||
return self.acompletion_stream_function(
|
||||
model=model,
|
||||
messages=messages,
|
||||
data=data,
|
||||
api_base=api_base,
|
||||
custom_prompt_dict=custom_prompt_dict,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
encoding=encoding,
|
||||
api_key=api_key,
|
||||
logging_obj=logging_obj,
|
||||
optional_params=optional_params,
|
||||
stream=stream,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
headers=headers,
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
streaming_decoder=streaming_decoder,
|
||||
fake_stream=fake_stream,
|
||||
)
|
||||
else:
|
||||
return self.acompletion_function(
|
||||
model=model,
|
||||
messages=messages,
|
||||
data=data,
|
||||
api_base=api_base,
|
||||
custom_prompt_dict=custom_prompt_dict,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
encoding=encoding,
|
||||
api_key=api_key,
|
||||
logging_obj=logging_obj,
|
||||
optional_params=optional_params,
|
||||
stream=stream,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
headers=headers,
|
||||
timeout=timeout,
|
||||
base_model=base_model,
|
||||
client=client,
|
||||
json_mode=json_mode,
|
||||
)
|
||||
else:
|
||||
## COMPLETION CALL
|
||||
if stream is True:
|
||||
completion_stream = make_sync_call(
|
||||
client=(
|
||||
client
|
||||
if client is not None and isinstance(client, HTTPHandler)
|
||||
else None
|
||||
),
|
||||
api_base=api_base,
|
||||
headers=headers,
|
||||
data=json.dumps(data),
|
||||
model=model,
|
||||
messages=messages,
|
||||
logging_obj=logging_obj,
|
||||
streaming_decoder=streaming_decoder,
|
||||
fake_stream=fake_stream,
|
||||
timeout=timeout,
|
||||
)
|
||||
# completion_stream.__iter__()
|
||||
return CustomStreamWrapper(
|
||||
completion_stream=completion_stream,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
else:
|
||||
if client is None or not isinstance(client, HTTPHandler):
|
||||
client = HTTPHandler(timeout=timeout) # type: ignore
|
||||
try:
|
||||
response = client.post(
|
||||
url=api_base, headers=headers, data=json.dumps(data)
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise OpenAILikeError(
|
||||
status_code=e.response.status_code,
|
||||
message=e.response.text,
|
||||
)
|
||||
except httpx.TimeoutException:
|
||||
raise OpenAILikeError(
|
||||
status_code=408, message="Timeout error occurred."
|
||||
)
|
||||
except Exception as e:
|
||||
raise OpenAILikeError(status_code=500, message=str(e))
|
||||
return OpenAILikeChatConfig._transform_response(
|
||||
model=model,
|
||||
response=response,
|
||||
model_response=model_response,
|
||||
stream=stream,
|
||||
logging_obj=logging_obj,
|
||||
optional_params=optional_params,
|
||||
api_key=api_key,
|
||||
data=data,
|
||||
messages=messages,
|
||||
print_verbose=print_verbose,
|
||||
encoding=encoding,
|
||||
json_mode=json_mode,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
base_model=base_model,
|
||||
)
|
||||
@@ -0,0 +1,179 @@
|
||||
"""
|
||||
OpenAI-like chat completion transformation
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantMessage
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
LiteLLMLoggingObj = Any
|
||||
|
||||
|
||||
class OpenAILikeChatConfig(OpenAIGPTConfig):
|
||||
def _get_openai_compatible_provider_info(
|
||||
self,
|
||||
api_base: Optional[str],
|
||||
api_key: Optional[str],
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
api_base = api_base or get_secret_str("OPENAI_LIKE_API_BASE") # type: ignore
|
||||
dynamic_api_key = (
|
||||
api_key or get_secret_str("OPENAI_LIKE_API_KEY") or ""
|
||||
) # vllm does not require an api key
|
||||
return api_base, dynamic_api_key
|
||||
|
||||
@staticmethod
|
||||
def _json_mode_convert_tool_response_to_message(
|
||||
message: ChatCompletionAssistantMessage, json_mode: bool
|
||||
) -> ChatCompletionAssistantMessage:
|
||||
"""
|
||||
if json_mode is true, convert the returned tool call response to a content with json str
|
||||
|
||||
e.g. input:
|
||||
|
||||
{"role": "assistant", "tool_calls": [{"id": "call_5ms4", "type": "function", "function": {"name": "json_tool_call", "arguments": "{\"key\": \"question\", \"value\": \"What is the capital of France?\"}"}}]}
|
||||
|
||||
output:
|
||||
|
||||
{"role": "assistant", "content": "{\"key\": \"question\", \"value\": \"What is the capital of France?\"}"}
|
||||
"""
|
||||
if not json_mode:
|
||||
return message
|
||||
|
||||
_tool_calls = message.get("tool_calls")
|
||||
|
||||
if _tool_calls is None or len(_tool_calls) != 1:
|
||||
return message
|
||||
|
||||
message["content"] = _tool_calls[0]["function"].get("arguments") or ""
|
||||
message["tool_calls"] = None
|
||||
|
||||
return message
|
||||
|
||||
@staticmethod
|
||||
def _sanitize_usage_obj(response_json: dict) -> dict:
|
||||
"""
|
||||
Checks for a 'usage' object in the response and replaces any None token values with 0.
|
||||
This enforces OpenAI compatibility for providers that might return null.
|
||||
|
||||
This method is future-proof and sanitizes any key ending in '_tokens'.
|
||||
"""
|
||||
if "usage" in response_json and isinstance(response_json.get("usage"), dict):
|
||||
usage = response_json["usage"]
|
||||
# Iterate through all keys in the usage dictionary
|
||||
for key, value in usage.items():
|
||||
# Sanitize if the key ends with '_tokens' and its value is None
|
||||
if key.endswith("_tokens") and value is None:
|
||||
usage[key] = 0
|
||||
return response_json
|
||||
|
||||
@staticmethod
|
||||
def _transform_response(
|
||||
model: str,
|
||||
response: httpx.Response,
|
||||
model_response: ModelResponse,
|
||||
stream: bool,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
optional_params: dict,
|
||||
api_key: Optional[str],
|
||||
data: Union[dict, str],
|
||||
messages: List,
|
||||
print_verbose,
|
||||
encoding,
|
||||
json_mode: Optional[bool],
|
||||
custom_llm_provider: Optional[str],
|
||||
base_model: Optional[str],
|
||||
) -> ModelResponse:
|
||||
response_json = response.json()
|
||||
logging_obj.post_call(
|
||||
input=messages,
|
||||
api_key="",
|
||||
original_response=response_json,
|
||||
additional_args={"complete_input_dict": data},
|
||||
)
|
||||
|
||||
# Sanitize the usage object at the source
|
||||
response_json = OpenAILikeChatConfig._sanitize_usage_obj(response_json)
|
||||
|
||||
if json_mode:
|
||||
for choice in response_json["choices"]:
|
||||
message = (
|
||||
OpenAILikeChatConfig._json_mode_convert_tool_response_to_message(
|
||||
choice.get("message"), json_mode
|
||||
)
|
||||
)
|
||||
choice["message"] = message
|
||||
|
||||
returned_response = ModelResponse(**response_json)
|
||||
|
||||
if custom_llm_provider is not None:
|
||||
returned_response.model = (
|
||||
custom_llm_provider + "/" + (returned_response.model or "")
|
||||
)
|
||||
|
||||
if base_model is not None:
|
||||
returned_response._hidden_params["model"] = base_model
|
||||
return returned_response
|
||||
|
||||
def transform_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
model_response: ModelResponse,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
request_data: dict,
|
||||
messages: List[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: Any,
|
||||
api_key: Optional[str] = None,
|
||||
json_mode: Optional[bool] = None,
|
||||
) -> ModelResponse:
|
||||
return OpenAILikeChatConfig._transform_response(
|
||||
model=model,
|
||||
response=raw_response,
|
||||
model_response=model_response,
|
||||
stream=optional_params.get("stream", False),
|
||||
logging_obj=logging_obj,
|
||||
optional_params=optional_params,
|
||||
api_key=api_key,
|
||||
data=request_data,
|
||||
messages=messages,
|
||||
print_verbose=None,
|
||||
encoding=None,
|
||||
json_mode=json_mode,
|
||||
custom_llm_provider=None,
|
||||
base_model=None,
|
||||
)
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
optional_params: dict,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
replace_max_completion_tokens_with_max_tokens: bool = True,
|
||||
) -> dict:
|
||||
mapped_params = super().map_openai_params(
|
||||
non_default_params, optional_params, model, drop_params
|
||||
)
|
||||
if (
|
||||
"max_completion_tokens" in non_default_params
|
||||
and replace_max_completion_tokens_with_max_tokens
|
||||
):
|
||||
mapped_params["max_tokens"] = non_default_params[
|
||||
"max_completion_tokens"
|
||||
] # most openai-compatible providers support 'max_tokens' not 'max_completion_tokens'
|
||||
mapped_params.pop("max_completion_tokens", None)
|
||||
|
||||
return mapped_params
|
||||
@@ -0,0 +1,56 @@
|
||||
from typing import Literal, Optional, Tuple
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
class OpenAILikeError(Exception):
|
||||
def __init__(self, status_code, message):
|
||||
self.status_code = status_code
|
||||
self.message = message
|
||||
self.request = httpx.Request(method="POST", url="https://www.litellm.ai")
|
||||
self.response = httpx.Response(status_code=status_code, request=self.request)
|
||||
super().__init__(
|
||||
self.message
|
||||
) # Call the base class constructor with the parameters it needs
|
||||
|
||||
|
||||
class OpenAILikeBase:
|
||||
def __init__(self, **kwargs):
|
||||
pass
|
||||
|
||||
def _validate_environment(
|
||||
self,
|
||||
api_key: Optional[str],
|
||||
api_base: Optional[str],
|
||||
endpoint_type: Literal["chat_completions", "embeddings"],
|
||||
headers: Optional[dict],
|
||||
custom_endpoint: Optional[bool],
|
||||
) -> Tuple[str, dict]:
|
||||
if api_key is None and headers is None:
|
||||
raise OpenAILikeError(
|
||||
status_code=400,
|
||||
message="Missing API Key - A call is being made to LLM Provider but no key is set either in the environment variables ({LLM_PROVIDER}_API_KEY) or via params",
|
||||
)
|
||||
|
||||
if api_base is None:
|
||||
raise OpenAILikeError(
|
||||
status_code=400,
|
||||
message="Missing API Base - A call is being made to LLM Provider but no api base is set either in the environment variables ({LLM_PROVIDER}_API_KEY) or via params",
|
||||
)
|
||||
|
||||
if headers is None:
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
if (
|
||||
api_key is not None and "Authorization" not in headers
|
||||
): # [TODO] remove 'validate_environment' from OpenAI base. should use llm providers config for this only.
|
||||
headers.update({"Authorization": "Bearer {}".format(api_key)})
|
||||
|
||||
if not custom_endpoint:
|
||||
if endpoint_type == "chat_completions":
|
||||
api_base = "{}/chat/completions".format(api_base)
|
||||
elif endpoint_type == "embeddings":
|
||||
api_base = "{}/embeddings".format(api_base)
|
||||
return api_base, headers
|
||||
@@ -0,0 +1,229 @@
|
||||
"""
|
||||
Dynamic configuration class generator for JSON-based providers.
|
||||
"""
|
||||
|
||||
from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, overload
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
handle_messages_with_content_list_to_str_conversion,
|
||||
)
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
from .json_loader import SimpleProviderConfig
|
||||
|
||||
|
||||
def create_config_class(provider: SimpleProviderConfig):
|
||||
"""Generate config class dynamically from JSON configuration"""
|
||||
|
||||
# Choose base class
|
||||
base_class: type = (
|
||||
OpenAIGPTConfig if provider.base_class == "openai_gpt" else OpenAILikeChatConfig
|
||||
)
|
||||
|
||||
class JSONProviderConfig(base_class): # type: ignore[valid-type,misc]
|
||||
@overload
|
||||
def _transform_messages(
|
||||
self, messages: List[AllMessageValues], model: str, is_async: Literal[True]
|
||||
) -> Coroutine[Any, Any, List[AllMessageValues]]:
|
||||
...
|
||||
|
||||
@overload
|
||||
def _transform_messages(
|
||||
self,
|
||||
messages: List[AllMessageValues],
|
||||
model: str,
|
||||
is_async: Literal[False] = False,
|
||||
) -> List[AllMessageValues]:
|
||||
...
|
||||
|
||||
def _transform_messages(
|
||||
self, messages: List[AllMessageValues], model: str, is_async: bool = False
|
||||
) -> Union[List[AllMessageValues], Coroutine[Any, Any, List[AllMessageValues]]]:
|
||||
"""Transform messages based on special_handling config"""
|
||||
|
||||
# Handle content list to string conversion if configured
|
||||
if provider.special_handling.get("convert_content_list_to_string"):
|
||||
messages = handle_messages_with_content_list_to_str_conversion(messages)
|
||||
|
||||
if is_async:
|
||||
return super()._transform_messages(
|
||||
messages=messages, model=model, is_async=True
|
||||
)
|
||||
else:
|
||||
return super()._transform_messages(
|
||||
messages=messages, model=model, is_async=False
|
||||
)
|
||||
|
||||
def _get_openai_compatible_provider_info(
|
||||
self, api_base: Optional[str], api_key: Optional[str]
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""Get API base and key from JSON config"""
|
||||
|
||||
# Resolve base URL
|
||||
resolved_base = api_base
|
||||
if not resolved_base and provider.api_base_env:
|
||||
resolved_base = get_secret_str(provider.api_base_env)
|
||||
if not resolved_base:
|
||||
resolved_base = provider.base_url
|
||||
|
||||
# Resolve API key
|
||||
resolved_key = api_key or get_secret_str(provider.api_key_env)
|
||||
|
||||
return resolved_base, resolved_key
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: Optional[str],
|
||||
api_key: Optional[str],
|
||||
model: str,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
stream: Optional[bool] = None,
|
||||
) -> str:
|
||||
"""Build complete URL for the API endpoint"""
|
||||
if not api_base:
|
||||
api_base = provider.base_url
|
||||
|
||||
if api_base is None:
|
||||
raise ValueError(f"api_base is required for provider {provider.slug}")
|
||||
|
||||
if not api_base.endswith("/chat/completions"):
|
||||
api_base = f"{api_base}/chat/completions"
|
||||
|
||||
return api_base
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
"""Get supported OpenAI params, excluding tool-related params for models
|
||||
that don't support function calling."""
|
||||
from litellm.utils import supports_function_calling
|
||||
|
||||
supported_params = super().get_supported_openai_params(model=model)
|
||||
|
||||
_supports_fc = supports_function_calling(
|
||||
model=model, custom_llm_provider=provider.slug
|
||||
)
|
||||
|
||||
if not _supports_fc:
|
||||
tool_params = [
|
||||
"tools",
|
||||
"tool_choice",
|
||||
"function_call",
|
||||
"functions",
|
||||
"parallel_tool_calls",
|
||||
]
|
||||
for param in tool_params:
|
||||
if param in supported_params:
|
||||
supported_params.remove(param)
|
||||
verbose_logger.debug(
|
||||
f"Model {model} on provider {provider.slug} does not support "
|
||||
f"function calling — removed tool-related params from supported params."
|
||||
)
|
||||
|
||||
return supported_params
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
optional_params: dict,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
"""Apply parameter mappings and constraints"""
|
||||
|
||||
supported_params = self.get_supported_openai_params(model)
|
||||
|
||||
# Apply supported params
|
||||
for param, value in non_default_params.items():
|
||||
# Check parameter mappings first
|
||||
if param in provider.param_mappings:
|
||||
optional_params[provider.param_mappings[param]] = value
|
||||
elif param in supported_params:
|
||||
optional_params[param] = value
|
||||
|
||||
# Apply temperature constraints if present
|
||||
if "temperature" in optional_params:
|
||||
temp = optional_params["temperature"]
|
||||
constraints = provider.constraints
|
||||
|
||||
# Clamp to max
|
||||
if "temperature_max" in constraints:
|
||||
temp = min(temp, constraints["temperature_max"])
|
||||
|
||||
# Clamp to min
|
||||
if "temperature_min" in constraints:
|
||||
temp = max(temp, constraints["temperature_min"])
|
||||
|
||||
# Special case: temperature_min_with_n_gt_1
|
||||
if "temperature_min_with_n_gt_1" in constraints:
|
||||
n = optional_params.get("n", 1)
|
||||
if n > 1 and temp < constraints["temperature_min_with_n_gt_1"]:
|
||||
temp = constraints["temperature_min_with_n_gt_1"]
|
||||
|
||||
optional_params["temperature"] = temp
|
||||
|
||||
return optional_params
|
||||
|
||||
@property
|
||||
def custom_llm_provider(self) -> Optional[str]:
|
||||
return provider.slug
|
||||
|
||||
return JSONProviderConfig
|
||||
|
||||
|
||||
_responses_config_cache: dict = {}
|
||||
|
||||
|
||||
def create_responses_config_class(provider: SimpleProviderConfig):
|
||||
"""Generate a Responses API config class dynamically from JSON configuration.
|
||||
|
||||
Parallel to create_config_class() but for /v1/responses endpoints.
|
||||
Classes are cached per provider slug to avoid regeneration on every request.
|
||||
"""
|
||||
if provider.slug in _responses_config_cache:
|
||||
return _responses_config_cache[provider.slug]
|
||||
|
||||
from litellm.llms.openai_like.responses.transformation import (
|
||||
OpenAILikeResponsesConfig,
|
||||
)
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
class JSONProviderResponsesConfig(OpenAILikeResponsesConfig):
|
||||
@property
|
||||
def custom_llm_provider(self): # type: ignore[override]
|
||||
return provider.slug
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
model: str,
|
||||
litellm_params: Optional[GenericLiteLLMParams],
|
||||
) -> dict:
|
||||
litellm_params = litellm_params or GenericLiteLLMParams()
|
||||
api_key = litellm_params.api_key or get_secret_str(provider.api_key_env)
|
||||
if api_key:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
return headers
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: Optional[str],
|
||||
litellm_params: dict,
|
||||
) -> str:
|
||||
if not api_base:
|
||||
if provider.api_base_env:
|
||||
api_base = get_secret_str(provider.api_base_env)
|
||||
if not api_base:
|
||||
api_base = provider.base_url
|
||||
|
||||
if api_base is None:
|
||||
raise ValueError(f"api_base is required for provider {provider.slug}")
|
||||
|
||||
api_base = api_base.rstrip("/")
|
||||
return f"{api_base}/responses"
|
||||
|
||||
_responses_config_cache[provider.slug] = JSONProviderResponsesConfig
|
||||
return JSONProviderResponsesConfig
|
||||
@@ -0,0 +1,156 @@
|
||||
# What is this?
|
||||
## Handler file for OpenAI-like endpoints.
|
||||
## Allows jina ai embedding calls - which don't allow 'encoding_format' in payload.
|
||||
|
||||
import json
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
HTTPHandler,
|
||||
get_async_httpx_client,
|
||||
)
|
||||
from litellm.types.utils import EmbeddingResponse
|
||||
|
||||
from ..common_utils import OpenAILikeBase, OpenAILikeError
|
||||
|
||||
|
||||
class OpenAILikeEmbeddingHandler(OpenAILikeBase):
|
||||
def __init__(self, **kwargs):
|
||||
pass
|
||||
|
||||
async def aembedding(
|
||||
self,
|
||||
input: list,
|
||||
data: dict,
|
||||
model_response: EmbeddingResponse,
|
||||
timeout: float,
|
||||
api_key: str,
|
||||
api_base: str,
|
||||
logging_obj,
|
||||
headers: dict,
|
||||
client=None,
|
||||
) -> EmbeddingResponse:
|
||||
response = None
|
||||
try:
|
||||
if client is None or not isinstance(client, AsyncHTTPHandler):
|
||||
async_client = get_async_httpx_client(
|
||||
llm_provider=litellm.LlmProviders.OPENAI,
|
||||
params={"timeout": timeout},
|
||||
)
|
||||
else:
|
||||
async_client = client
|
||||
try:
|
||||
response = await async_client.post(
|
||||
api_base,
|
||||
headers=headers,
|
||||
data=json.dumps(data),
|
||||
) # type: ignore
|
||||
|
||||
response.raise_for_status()
|
||||
|
||||
response_json = response.json()
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise OpenAILikeError(
|
||||
status_code=e.response.status_code,
|
||||
message=e.response.text if e.response else str(e),
|
||||
)
|
||||
except httpx.TimeoutException:
|
||||
raise OpenAILikeError(
|
||||
status_code=408, message="Timeout error occurred."
|
||||
)
|
||||
except Exception as e:
|
||||
raise OpenAILikeError(status_code=500, message=str(e))
|
||||
|
||||
## LOGGING
|
||||
logging_obj.post_call(
|
||||
input=input,
|
||||
api_key=api_key,
|
||||
additional_args={"complete_input_dict": data},
|
||||
original_response=response_json,
|
||||
)
|
||||
return EmbeddingResponse(**response_json)
|
||||
except Exception as e:
|
||||
## LOGGING
|
||||
logging_obj.post_call(
|
||||
input=input,
|
||||
api_key=api_key,
|
||||
original_response=str(e),
|
||||
)
|
||||
raise e
|
||||
|
||||
def embedding(
|
||||
self,
|
||||
model: str,
|
||||
input: list,
|
||||
timeout: float,
|
||||
logging_obj,
|
||||
api_key: Optional[str],
|
||||
api_base: Optional[str],
|
||||
optional_params: dict,
|
||||
model_response: Optional[EmbeddingResponse] = None,
|
||||
client=None,
|
||||
aembedding=None,
|
||||
custom_endpoint: Optional[bool] = None,
|
||||
headers: Optional[dict] = None,
|
||||
) -> EmbeddingResponse:
|
||||
api_base, headers = self._validate_environment(
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
endpoint_type="embeddings",
|
||||
headers=headers,
|
||||
custom_endpoint=custom_endpoint,
|
||||
)
|
||||
model = model
|
||||
filtered_optional_params = {
|
||||
k: v for k, v in optional_params.items() if v not in (None, "")
|
||||
}
|
||||
data = {"model": model, "input": input, **filtered_optional_params}
|
||||
|
||||
## LOGGING
|
||||
logging_obj.pre_call(
|
||||
input=input,
|
||||
api_key=api_key,
|
||||
additional_args={"complete_input_dict": data, "api_base": api_base},
|
||||
)
|
||||
|
||||
if aembedding is True:
|
||||
return self.aembedding(data=data, input=input, logging_obj=logging_obj, model_response=model_response, api_base=api_base, api_key=api_key, timeout=timeout, client=client, headers=headers) # type: ignore
|
||||
if client is None or isinstance(client, AsyncHTTPHandler):
|
||||
self.client = HTTPHandler(timeout=timeout) # type: ignore
|
||||
else:
|
||||
self.client = client
|
||||
|
||||
## EMBEDDING CALL
|
||||
try:
|
||||
response = self.client.post(
|
||||
api_base,
|
||||
headers=headers,
|
||||
data=json.dumps(data),
|
||||
) # type: ignore
|
||||
|
||||
response.raise_for_status() # type: ignore
|
||||
|
||||
response_json = response.json() # type: ignore
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise OpenAILikeError(
|
||||
status_code=e.response.status_code,
|
||||
message=e.response.text,
|
||||
)
|
||||
except httpx.TimeoutException:
|
||||
raise OpenAILikeError(status_code=408, message="Timeout error occurred.")
|
||||
except Exception as e:
|
||||
raise OpenAILikeError(status_code=500, message=str(e))
|
||||
|
||||
## LOGGING
|
||||
logging_obj.post_call(
|
||||
input=input,
|
||||
api_key=api_key,
|
||||
additional_args={"complete_input_dict": data},
|
||||
original_response=response_json,
|
||||
)
|
||||
|
||||
return litellm.EmbeddingResponse(**response_json)
|
||||
@@ -0,0 +1,85 @@
|
||||
"""
|
||||
JSON-based provider configuration loader for OpenAI-compatible providers.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
|
||||
class SimpleProviderConfig:
|
||||
"""Simple data class for JSON provider config"""
|
||||
|
||||
def __init__(self, slug: str, data: dict):
|
||||
self.slug = slug
|
||||
self.base_url = data["base_url"]
|
||||
self.api_key_env = data["api_key_env"]
|
||||
self.api_base_env = data.get("api_base_env")
|
||||
self.base_class = data.get("base_class", "openai_gpt")
|
||||
self.param_mappings = data.get("param_mappings", {})
|
||||
self.constraints = data.get("constraints", {})
|
||||
self.special_handling = data.get("special_handling", {})
|
||||
self.supported_endpoints = data.get("supported_endpoints", [])
|
||||
|
||||
|
||||
class JSONProviderRegistry:
|
||||
"""Load providers from JSON once on import"""
|
||||
|
||||
_providers: Dict[str, SimpleProviderConfig] = {}
|
||||
_loaded = False
|
||||
|
||||
@classmethod
|
||||
def load(cls):
|
||||
"""Load providers from JSON configuration file"""
|
||||
if cls._loaded:
|
||||
return
|
||||
|
||||
json_path = Path(__file__).parent / "providers.json"
|
||||
|
||||
if not json_path.exists():
|
||||
# No JSON file yet, that's okay
|
||||
cls._loaded = True
|
||||
return
|
||||
|
||||
try:
|
||||
with open(json_path) as f:
|
||||
data = json.load(f)
|
||||
|
||||
for slug, config in data.items():
|
||||
cls._providers[slug] = SimpleProviderConfig(slug, config)
|
||||
|
||||
cls._loaded = True
|
||||
except Exception as e:
|
||||
verbose_logger.warning(
|
||||
f"Warning: Failed to load JSON provider configs: {e}"
|
||||
)
|
||||
cls._loaded = True
|
||||
|
||||
@classmethod
|
||||
def get(cls, slug: str) -> Optional[SimpleProviderConfig]:
|
||||
"""Get a provider configuration by slug"""
|
||||
return cls._providers.get(slug)
|
||||
|
||||
@classmethod
|
||||
def exists(cls, slug: str) -> bool:
|
||||
"""Check if a provider is defined via JSON"""
|
||||
return slug in cls._providers
|
||||
|
||||
@classmethod
|
||||
def supports_responses_api(cls, slug: str) -> bool:
|
||||
"""Check if a JSON provider supports the Responses API"""
|
||||
provider = cls._providers.get(slug)
|
||||
if provider is None:
|
||||
return False
|
||||
return "/v1/responses" in provider.supported_endpoints
|
||||
|
||||
@classmethod
|
||||
def list_providers(cls) -> list:
|
||||
"""List all registered provider slugs"""
|
||||
return list(cls._providers.keys())
|
||||
|
||||
|
||||
# Load on import
|
||||
JSONProviderRegistry.load()
|
||||
@@ -0,0 +1,105 @@
|
||||
{
|
||||
"publicai": {
|
||||
"base_url": "https://api.publicai.co/v1",
|
||||
"api_key_env": "PUBLICAI_API_KEY",
|
||||
"api_base_env": "PUBLICAI_API_BASE",
|
||||
"base_class": "openai_gpt",
|
||||
"param_mappings": {
|
||||
"max_completion_tokens": "max_tokens"
|
||||
},
|
||||
"special_handling": {
|
||||
"convert_content_list_to_string": true
|
||||
}
|
||||
},
|
||||
"helicone": {
|
||||
"base_url": "https://ai-gateway.helicone.ai/",
|
||||
"api_key_env": "HELICONE_API_KEY"
|
||||
},
|
||||
"veniceai": {
|
||||
"base_url": "https://api.venice.ai/api/v1",
|
||||
"api_key_env": "VENICE_AI_API_KEY"
|
||||
},
|
||||
"xiaomi_mimo": {
|
||||
"base_url": "https://api.xiaomimimo.com/v1",
|
||||
"api_key_env": "XIAOMI_MIMO_API_KEY",
|
||||
"param_mappings": {
|
||||
"max_completion_tokens": "max_tokens"
|
||||
}
|
||||
},
|
||||
"scaleway": {
|
||||
"base_url": "https://api.scaleway.ai/v1",
|
||||
"api_key_env": "SCW_SECRET_KEY"
|
||||
},
|
||||
"synthetic": {
|
||||
"base_url": "https://api.synthetic.new/openai/v1",
|
||||
"api_key_env": "SYNTHETIC_API_KEY",
|
||||
"param_mappings": {
|
||||
"max_completion_tokens": "max_tokens"
|
||||
}
|
||||
},
|
||||
"apertis": {
|
||||
"base_url": "https://api.stima.tech/v1",
|
||||
"api_key_env": "STIMA_API_KEY",
|
||||
"param_mappings": {
|
||||
"max_completion_tokens": "max_tokens"
|
||||
}
|
||||
},
|
||||
"nano-gpt": {
|
||||
"base_url": "https://nano-gpt.com/api/v1",
|
||||
"api_key_env": "NANOGPT_API_KEY",
|
||||
"param_mappings": {
|
||||
"max_completion_tokens": "max_tokens"
|
||||
}
|
||||
},
|
||||
"poe": {
|
||||
"base_url": "https://api.poe.com/v1",
|
||||
"api_key_env": "POE_API_KEY",
|
||||
"param_mappings": {
|
||||
"max_completion_tokens": "max_tokens"
|
||||
}
|
||||
},
|
||||
"chutes": {
|
||||
"base_url": "https://llm.chutes.ai/v1/",
|
||||
"api_key_env": "CHUTES_API_KEY",
|
||||
"param_mappings": {
|
||||
"max_completion_tokens": "max_tokens"
|
||||
}
|
||||
},
|
||||
"abliteration": {
|
||||
"base_url": "https://api.abliteration.ai/v1",
|
||||
"api_key_env": "ABLITERATION_API_KEY"
|
||||
},
|
||||
"llamagate": {
|
||||
"base_url": "https://api.llamagate.dev/v1",
|
||||
"api_key_env": "LLAMAGATE_API_KEY",
|
||||
"param_mappings": {
|
||||
"max_completion_tokens": "max_tokens"
|
||||
}
|
||||
},
|
||||
"gmi": {
|
||||
"base_url": "https://api.gmi-serving.com/v1",
|
||||
"api_key_env": "GMI_API_KEY"
|
||||
},
|
||||
"sarvam": {
|
||||
"base_url": "https://api.sarvam.ai/v1",
|
||||
"api_key_env": "SARVAM_API_KEY",
|
||||
"base_class": "openai_gpt",
|
||||
"param_mappings": {
|
||||
"max_completion_tokens": "max_tokens"
|
||||
},
|
||||
"headers": {
|
||||
"api-subscription-key": "{api_key}"
|
||||
}
|
||||
},
|
||||
"assemblyai": {
|
||||
"base_url": "https://llm-gateway.assemblyai.com/v1",
|
||||
"api_key_env": "ASSEMBLYAI_API_KEY"
|
||||
},
|
||||
"charity_engine": {
|
||||
"base_url": "https://api.charityengine.services/remotejobs/v2/inference",
|
||||
"api_key_env": "CHARITY_ENGINE_API_KEY",
|
||||
"param_mappings": {
|
||||
"max_completion_tokens": "max_tokens"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
from litellm.llms.openai_like.responses.transformation import (
|
||||
OpenAILikeResponsesConfig,
|
||||
)
|
||||
|
||||
__all__ = ["OpenAILikeResponsesConfig"]
|
||||
@@ -0,0 +1,51 @@
|
||||
"""
|
||||
OpenAI-like Responses API transformation.
|
||||
|
||||
Base class for JSON-declared providers that support the /v1/responses endpoint.
|
||||
Inherits everything from OpenAIResponsesAPIConfig; subclasses only override
|
||||
provider-specific resolution (slug, API key env var, base URL).
|
||||
"""
|
||||
|
||||
from typing import Optional, Union
|
||||
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
|
||||
class OpenAILikeResponsesConfig(OpenAIResponsesAPIConfig):
|
||||
"""
|
||||
Responses API config for OpenAI-compatible providers declared via JSON.
|
||||
|
||||
Concrete per-provider classes are generated dynamically in dynamic_config.py.
|
||||
This base provides the three overridable hooks that the dynamic generator
|
||||
fills in: custom_llm_provider, validate_environment, get_complete_url.
|
||||
"""
|
||||
|
||||
@property
|
||||
def custom_llm_provider(self) -> Union[str, LlmProviders]: # type: ignore[override]
|
||||
return "openai_like"
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
model: str,
|
||||
litellm_params: Optional[GenericLiteLLMParams],
|
||||
) -> dict:
|
||||
litellm_params = litellm_params or GenericLiteLLMParams()
|
||||
api_key = litellm_params.api_key or get_secret_str("OPENAI_LIKE_API_KEY")
|
||||
if api_key:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
return headers
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: Optional[str],
|
||||
litellm_params: dict,
|
||||
) -> str:
|
||||
api_base = api_base or get_secret_str("OPENAI_LIKE_API_BASE")
|
||||
if not api_base:
|
||||
raise ValueError("api_base is required for openai_like provider")
|
||||
api_base = api_base.rstrip("/")
|
||||
return f"{api_base}/responses"
|
||||
Reference in New Issue
Block a user