change name to context_length
This commit is contained in:
parent
61a5254115
commit
b4de2e9189
@ -206,29 +206,29 @@ curl http://localhost:11434/v1/embeddings \
|
|||||||
|
|
||||||
## Extra arguments
|
## Extra arguments
|
||||||
|
|
||||||
### Setting context window size
|
### Setting context length
|
||||||
- `context_window` parameter can be used to set the context window for the model
|
- `context_length` parameter can be used to set the context length for the model
|
||||||
|
|
||||||
#### OpenAI python library
|
#### OpenAI python library
|
||||||
- OpenAI python library does not support setting context window size, however this can be set for Ollama through the `extra_body` parameter
|
- OpenAI python library does not support setting context length, however this can be set for Ollama through the `extra_body` parameter
|
||||||
|
|
||||||
```py
|
```py
|
||||||
completion = client.chat.completions.create(
|
completion = client.chat.completions.create(
|
||||||
model="llama3.1:8b",
|
model="llama3.1:8b",
|
||||||
messages=[{"role": "user", "content": "Say this is a test"}],
|
messages=[{"role": "user", "content": "Say this is a test"}],
|
||||||
extra_body={"context_window": 4096},
|
extra_body={"context_length": 4096},
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
#### OpenAI JavaScript library
|
#### OpenAI JavaScript library
|
||||||
- OpenAI JavaScript library does not support setting context window size, however this can be set for Ollama by passing `num_ctx` directly with a `@ts-expect-error` as an undocumented parameter in the OpenAI JavaScript library. [See documentation here](https://github.com/openai/openai-node?tab=readme-ov-file#making-customundocumented-requests)
|
- OpenAI JavaScript library does not support setting context length, however this can be set for Ollama by passing `context_length` directly with a `@ts-expect-error` as an undocumented parameter in the OpenAI JavaScript library. [See documentation here](https://github.com/openai/openai-node?tab=readme-ov-file#making-customundocumented-requests)
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
const chatCompletion = await openai.chat.completions.create({
|
const chatCompletion = await openai.chat.completions.create({
|
||||||
messages: [{ role: 'user', content: 'Say this is a test' }],
|
messages: [{ role: 'user', content: 'Say this is a test' }],
|
||||||
model: 'llama3.2',
|
model: 'llama3.2',
|
||||||
// @ts-expect-error context_window is an additional parameter
|
// @ts-expect-error context_length is an additional parameter
|
||||||
context_window: 4096,
|
context_length: 4096,
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -239,7 +239,7 @@ curl http://localhost:11434/v1/chat/completions \
|
|||||||
-d '{
|
-d '{
|
||||||
"model": "llama3.2",
|
"model": "llama3.2",
|
||||||
"messages": [{"role": "user", "content": "Say this is a test"}],
|
"messages": [{"role": "user", "content": "Say this is a test"}],
|
||||||
"context_window": 4096
|
"context_length": 4096
|
||||||
}'
|
}'
|
||||||
```
|
```
|
||||||
|
|
||||||
|
@ -95,7 +95,7 @@ type ChatCompletionRequest struct {
|
|||||||
TopP *float64 `json:"top_p"`
|
TopP *float64 `json:"top_p"`
|
||||||
ResponseFormat *ResponseFormat `json:"response_format"`
|
ResponseFormat *ResponseFormat `json:"response_format"`
|
||||||
Tools []api.Tool `json:"tools"`
|
Tools []api.Tool `json:"tools"`
|
||||||
ContextWindow *int `json:"context_window"`
|
ContextLength *int `json:"context_length"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ChatCompletion struct {
|
type ChatCompletion struct {
|
||||||
@ -478,9 +478,8 @@ func fromChatRequest(r ChatCompletionRequest) (*api.ChatRequest, error) {
|
|||||||
options["stop"] = stops
|
options["stop"] = stops
|
||||||
}
|
}
|
||||||
|
|
||||||
if r.ContextWindow != nil {
|
if r.ContextLength != nil {
|
||||||
slog.Info("context_window in if", "context_window", *r.ContextWindow)
|
options["num_ctx"] = *r.ContextLength
|
||||||
options["num_ctx"] = *r.ContextWindow
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Deprecated: MaxTokens is deprecated, use MaxCompletionTokens instead
|
// Deprecated: MaxTokens is deprecated, use MaxCompletionTokens instead
|
||||||
|
@ -315,11 +315,11 @@ func TestChatMiddleware(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "chat handler with context_window",
|
name: "chat handler with context_length",
|
||||||
body: `{
|
body: `{
|
||||||
"model": "test-model",
|
"model": "test-model",
|
||||||
"messages": [{"role": "user", "content": "Hello"}],
|
"messages": [{"role": "user", "content": "Hello"}],
|
||||||
"context_window": 4096
|
"context_length": 4096
|
||||||
}`,
|
}`,
|
||||||
req: api.ChatRequest{
|
req: api.ChatRequest{
|
||||||
Model: "test-model",
|
Model: "test-model",
|
||||||
|
Loading…
x
Reference in New Issue
Block a user