> ## Documentation Index
> Fetch the complete documentation index at: https://portkey-docs-mintlify-add-terraform-provider-docs-72855.mintlify.site/llms.txt
> Use this file to discover all available pages before exploring further.
# Gateway Config Object
The `config` object is used to configure API interactions with various providers. It supports multiple modes such as single provider access, load balancing between providers, and fallback strategies.
**The following JSON schema is used to validate the config object:**
```js theme={null}
{
$schema: 'http://json-schema.org/draft-07/schema#',
type: 'object',
properties: {
after_request_hooks: {
type: 'array',
items: {
properties: {
id: {
type: 'string',
},
type: {
type: 'string',
},
async: {
type: 'boolean',
},
on_fail: {
type: 'object',
properties: {
feedback: {
type: 'object',
properties: {
value: {
type: 'number',
},
weight: {
type: 'number',
},
metadata: {
type: 'object',
},
},
},
},
},
on_success: {
type: 'object',
properties: {
feedback: {
type: 'object',
properties: {
value: {
type: 'number',
},
weight: {
type: 'number',
},
metadata: {
type: 'object',
},
},
},
},
},
checks: {
type: 'array',
items: {
type: 'object',
properties: {
id: {
type: 'string',
},
parameters: {
type: 'object',
},
},
required: ['id', 'parameters'],
},
},
},
required: ['id'],
},
},
input_guardrails: {
type: 'array',
items: {
oneOf: [
{
type: 'object',
properties: {
id: {
type: 'string',
},
deny: {
type: 'boolean',
},
on_fail: {
type: 'object',
properties: {
feedback: {
type: 'object',
properties: {
value: {
type: 'number',
},
weight: {
type: 'number',
},
metadata: {
type: 'object',
},
},
},
},
},
on_success: {
type: 'object',
properties: {
feedback: {
type: 'object',
properties: {
value: {
type: 'number',
},
weight: {
type: 'number',
},
metadata: {
type: 'object',
},
},
},
},
},
async: {
type: 'boolean',
},
},
additionalProperties: {
type: 'object',
additionalProperties: true,
},
},
{
type: 'string',
},
],
},
},
output_guardrails: {
type: 'array',
items: {
oneOf: [
{
type: 'object',
properties: {
id: {
type: 'string',
},
deny: {
type: 'boolean',
},
on_fail: {
type: 'object',
properties: {
feedback: {
type: 'object',
properties: {
value: {
type: 'number',
},
weight: {
type: 'number',
},
metadata: {
type: 'object',
},
},
},
deny: {
type: 'boolean',
},
},
},
on_success: {
type: 'object',
properties: {
feedback: {
type: 'object',
properties: {
value: {
type: 'number',
},
weight: {
type: 'number',
},
metadata: {
type: 'object',
},
},
},
deny: {
type: 'boolean',
},
},
},
async: {
type: 'boolean',
},
},
additionalProperties: {
type: 'object',
additionalProperties: true,
},
},
{
type: 'string',
},
],
},
},
before_request_hooks: {
type: 'array',
items: {
properties: {
id: {
type: 'string',
},
type: {
type: 'string',
},
on_fail: {
type: 'object',
properties: {
feedback: {
type: 'object',
properties: {
value: {
type: 'number',
},
weight: {
type: 'number',
},
metadata: {
type: 'object',
},
},
},
deny: {
type: 'boolean',
},
},
},
on_success: {
type: 'object',
properties: {
feedback: {
type: 'object',
properties: {
value: {
type: 'number',
},
weight: {
type: 'number',
},
metadata: {
type: 'object',
},
},
},
deny: {
type: 'boolean',
},
},
},
checks: {
type: 'array',
items: {
type: 'object',
properties: {
id: {
type: 'string',
},
parameters: {
type: 'object',
},
},
required: ['id', 'parameters'],
},
},
},
required: ['id'],
},
},
strategy: {
type: 'object',
properties: {
mode: {
type: 'string',
enum: ['single', 'loadbalance', 'fallback', 'conditional'],
},
conditions: {
type: 'array',
items: {
type: 'object',
properties: {
query: {
type: 'object',
},
then: {
type: 'string',
},
},
required: ['query', 'then'],
},
},
default: {
type: 'string',
},
on_status_codes: {
type: 'array',
items: {
type: 'integer',
},
optional: true,
},
},
allOf: [
{
if: {
properties: {
mode: {
const: 'conditional',
},
},
},
then: {
required: ['conditions', 'default'],
},
},
],
required: ['mode'],
},
name: {
type: 'string',
},
strict_open_ai_compliance: {
type: 'boolean',
},
provider: {
type: 'string',
},
resource_name: {
type: 'string',
optional: true,
},
deployment_id: {
type: 'string',
optional: true,
},
api_version: {
type: 'string',
optional: true,
},
deployments: {
type: 'array',
optional: true,
items: {
type: 'object',
properties: {
deployment_id: {
type: 'string',
},
alias: {
type: 'string',
},
api_version: {
type: 'string',
},
is_default: {
type: 'boolean',
},
},
required: ['deployment_id', 'alias', 'api_version'],
},
},
override_params: {
type: 'object',
},
api_key: {
type: 'string',
},
virtual_key: {
type: 'string',
},
prompt_id: {
type: 'string',
},
request_timeout: {
type: 'integer',
},
cache: {
type: 'object',
properties: {
mode: {
type: 'string',
enum: ['simple', 'semantic'],
},
max_age: {
type: 'integer',
optional: true,
},
},
required: ['mode'],
},
cb_config: {
type: 'object',
properties: {
failure_threshold: {
type: 'number',
minimum: 1,
},
cooldown_interval: {
type: 'number',
minimum: 30000,
},
failure_status_codes: {
type: 'array',
items: {
type: 'integer',
},
optional: true,
},
},
required: ['failure_threshold', 'cooldown_interval'],
},
retry: {
type: 'object',
properties: {
attempts: {
type: 'integer',
},
use_retry_after_headers: {
type: 'boolean',
},
on_status_codes: {
type: 'array',
items: {
type: 'number',
},
optional: true,
},
},
required: ['attempts'],
},
weight: {
type: 'number',
},
on_status_codes: {
type: 'array',
items: {
type: 'integer',
},
},
custom_host: {
type: 'string',
},
forward_headers: {
type: 'array',
items: {
type: 'string',
},
},
targets: {
type: 'array',
items: {
$ref: '#',
},
},
aws_access_key_id: {
type: 'string',
},
aws_secret_access_key: {
type: 'string',
},
aws_region: {
type: 'string',
},
aws_session_token: {
type: 'string',
},
openai_organization: {
type: 'string',
},
openai_project: {
type: 'string',
},
vertex_project_id: {
type: 'string',
},
vertex_region: {
type: 'string',
},
vertex_service_account_json: {
type: 'object',
},
azure_region: {
type: 'string',
},
azure_deployment_name: {
type: 'string',
},
azure_deployment_type: {
type: 'string',
enum: ['serverless', 'managed'],
},
azure_endpoint_name: {
type: 'string',
},
azure_api_version: {
type: 'string',
},
},
anyOf: [
{
required: ['provider', 'api_key'],
},
{
required: ['provider', 'custom_host'],
},
{
required: ['virtual_key'],
},
{
required: ['strategy', 'targets'],
},
{
required: ['cache'],
},
{
required: ['retry'],
},
{
required: ['prompt_id'],
},
{
required: ['forward_headers'],
},
{
required: ['request_timeout'],
},
{
required: ['provider', 'aws_access_key_id', 'aws_secret_access_key'],
},
{
required: ['provider', 'vertex_region', 'vertex_service_account_json'],
},
{
required: ['provider', 'vertex_region', 'vertex_project_id'],
},
{
required: [
'provider',
'azure_deployment_name',
'azure_deployment_type',
'azure_region',
'azure_api_version',
],
},
{
required: ['provider', 'azure_endpoint_name', 'azure_deployment_type'],
},
{
required: ['after_request_hooks'],
},
{
required: ['before_request_hooks'],
},
{
required: ['input_guardrails'],
},
{
required: ['output_guardrails'],
},
],
additionalProperties: false,
}
```
## Example Configs
```js theme={null}
// Simple config with cache and retry
{
"virtual_key": "***", // Your Virtual Key
"cache": { // Optional
"mode": "semantic",
"max_age": 10000
},
"retry": { // Optional
"attempts": 5,
"on_status_codes": []
}
}
// Load balancing with 2 OpenAI keys
{
"strategy": {
"mode": "loadbalance"
},
"targets": [
{
"provider": "openai",
"api_key": "sk-***"
},
{
"provider": "openai",
"api_key": "sk-***"
}
]
}
```
You can find more examples of schemas [below](/api-reference/inference-api/config-object#examples).
## Schema Details
| Key Name | Description | Type | Required | Enum Values | Additional Info |
| ----------------- | ------------------------------------------------------------ | ---------------- | --------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------- |
| `strategy` | Operational strategy for the config or any individual target | object | Yes (if no `provider` or `virtual_key`) | - | See Strategy Object Details |
| `provider` | Name of the service provider | string | Yes (if no `mode` or `virtual_key`) | "openai", "anthropic", "azure-openai", "anyscale", "cohere" | - |
| `api_key` | API key for the service provider | string | Yes (if `provider` is specified) | - | - |
| `virtual_key` | Virtual key identifier | string | Yes (if no `mode` or `provider`) | - | - |
| `cache` | Caching configuration | object | No | - | See Cache Object Details |
| `retry` | Retry configuration | object | No | - | See Retry Object Details |
| `weight` | Weight for load balancing | number | No | - | Used in `loadbalance` mode |
| `on_status_codes` | Status codes triggering fallback | array of strings | No | - | Used in `fallback` mode |
| `targets` | List of target configurations | array | Yes (if `mode` is specified) | - | Each item follows the config schema |
| `request_timeout` | Request timeout configuration | number | No | - | - |
| `custom_host` | Route to privately hosted model | string | No | - | Used in combination with `provider` + `api_key` |
| `forward_headers` | Forward sensitive headers directly | array of strings | No | - | - |
| `override_params` | Pass model name and other hyper parameters | object | No | "model", "temperature", "frequency\_penalty", "logit\_bias", "logprobs", "top\_logprobs", "max\_tokens", "n", "presence\_penalty", "response\_format", "seed", "stop", "top\_p", etc. | Pass everything that's typically part of the payload |
### Strategy Object Details
| Key Name | Description | Type | Required | Enum Values | Additional Info |
| ----------------- | -------------------------------------------------------------------------------------------- | ---------------- | -------- | ------------------------- | --------------- |
| `mode` | strategy mode for the config | string | Yes | "loadbalance", "fallback" | |
| `on_status_codes` | status codes to apply the strategy. This field is only used when strategy mode is "fallback" | array of numbers | No | | Optional |
### Cache Object Details
| Key Name | Description | Type | Required | Enum Values | Additional Info |
| --------- | ----------------------------- | ------- | -------- | -------------------- | --------------- |
| `mode` | Cache mode | string | Yes | "simple", "semantic" | - |
| `max_age` | Maximum age for cache entries | integer | No | - | Optional |
### Retry Object Details
| Key Name | Description | Type | Required | Enum Values | Additional Info |
| ------------------------- | -------------------------------------------------------------------- | ---------------- | -------------- | ----------- | --------------- |
| `attempts` | Number of retry attempts | integer | Yes | - | - |
| `on_status_codes` | Status codes to trigger retries | array of strings | No | - | Optional |
| `use_retry_after_headers` | Whether to respect provider's Retry-After and Retry-After-ms headers | boolean | Default: false | | |
### Circuit Breaker Object Details
| Key Name | Description | Type | Required | Enum Values | Additional Info |
| ---------------------- | ------------------------------------------------------ | ----------------- | -------- | ----------- | --------------------------------------- |
| `failure_threshold` | Number of failures after which the circuit opens | number | Yes | - | Minimum value: 1 |
| `cooldown_interval` | Time (in milliseconds) to wait before allowing retries | number | Yes | - | Minimum value: 30000 (30 seconds) |
| `failure_status_codes` | Specific HTTP status codes considered as failures | array of integers | No | - | Optional, defaults to status codes >500 |
### Cloud Provider Params (Azure OpenAI, Google Vertex, AWS Bedrock)
#### Azure OpenAI
| Key Name | Type | Required |
| --------------------- | ---------------------------- | -------- |
| `azure_resource_name` | string | No |
| `azure_deployment_id` | string | No |
| `azure_api_version` | string | No |
| `azure_model_name` | string | No |
| `Authorization` | string ("Bearer \$API\_KEY") | No |
#### Google Vertex AI
| Key Name | Type | Required |
| ------------------- | ------ | -------- |
| `vertex_project_id` | string | No |
| `vertex_region` | string | No |
#### AWS Bedrock
| Key Name | Type | Required |
| ----------------------- | ------ | -------- |
| `aws_access_key_id` | string | No |
| `aws_secret_access_key` | string | No |
| `aws_region` | string | No |
| `aws_session_token` | string | No |
### Notes
* The strategy `mode` key determines the operational mode of the config. If strategy `mode` is not specified, a single provider mode is assumed, requiring either `provider` and `api_key` or `virtual_key`.
* In `loadbalance` and `fallback` modes, the `targets` array specifies the configurations for each target.
* The `cache` and `retry` objects provide additional configurations for caching and retry policies, respectively.
## Examples
```json theme={null}
{
"provider": "openai",
"api_key": "sk-***"
}
```
```json theme={null}
{
"provider": "anthropic",
"api_key": "xxx",
"override_params": {
"model": "claude-3-sonnet-20240229",
"max_tokens": 512,
"temperature": 0
}
}
```
```json theme={null}
{
"virtual_key": "***"
}
```
```json theme={null}
{
"virtual_key": "***",
"cache": {
"mode": "semantic",
"max_age": 10000
},
"retry": {
"attempts": 5,
"on_status_codes": [429]
}
}
```
```json theme={null}
{
"strategy": {
"mode": "loadbalance"
},
"targets": [
{
"provider": "openai",
"api_key": "sk-***"
},
{
"provider": "openai",
"api_key": "sk-***"
}
]
}
```
```json theme={null}
{
"strategy": {
"mode": "loadbalance"
},
"targets": [
{
"provider": "openai",
"api_key": "sk-***"
},
{
"strategy": {
"mode": "fallback",
"on_status_codes": [429, 241]
},
"targets": [
{
"virtual_key": "***"
},
{
"virtual_key": "***"
}
]
}
]
}
```