change name to context_length

This commit is contained in:
ParthSareen 2025-02-07 11:48:22 -08:00
parent 61a5254115
commit b4de2e9189
3 changed files with 13 additions and 14 deletions

View File

@ -206,29 +206,29 @@ curl http://localhost:11434/v1/embeddings \
## Extra arguments
### Setting context window size
- `context_window` parameter can be used to set the context window for the model
### Setting context length
- `context_length` parameter can be used to set the context length for the model
#### OpenAI python library
- OpenAI python library does not support setting context window size, however this can be set for Ollama through the `extra_body` parameter
- OpenAI python library does not support setting context length, however this can be set for Ollama through the `extra_body` parameter
```py
completion = client.chat.completions.create(
model="llama3.1:8b",
messages=[{"role": "user", "content": "Say this is a test"}],
extra_body={"context_window": 4096},
extra_body={"context_length": 4096},
)
```
#### OpenAI JavaScript library
- OpenAI JavaScript library does not support setting context window size, however this can be set for Ollama by passing `num_ctx` directly with a `@ts-expect-error` as an undocumented parameter in the OpenAI JavaScript library. [See documentation here](https://github.com/openai/openai-node?tab=readme-ov-file#making-customundocumented-requests)
- OpenAI JavaScript library does not support setting context length, however this can be set for Ollama by passing `context_length` directly with a `@ts-expect-error` as an undocumented parameter in the OpenAI JavaScript library. [See documentation here](https://github.com/openai/openai-node?tab=readme-ov-file#making-customundocumented-requests)
```ts
const chatCompletion = await openai.chat.completions.create({
messages: [{ role: 'user', content: 'Say this is a test' }],
model: 'llama3.2',
// @ts-expect-error context_window is an additional parameter
context_window: 4096,
// @ts-expect-error context_length is an additional parameter
context_length: 4096,
})
```
@ -239,7 +239,7 @@ curl http://localhost:11434/v1/chat/completions \
-d '{
"model": "llama3.2",
"messages": [{"role": "user", "content": "Say this is a test"}],
"context_window": 4096
"context_length": 4096
}'
```

View File

@ -95,7 +95,7 @@ type ChatCompletionRequest struct {
TopP *float64 `json:"top_p"`
ResponseFormat *ResponseFormat `json:"response_format"`
Tools []api.Tool `json:"tools"`
ContextWindow *int `json:"context_window"`
ContextLength *int `json:"context_length"`
}
type ChatCompletion struct {
@ -478,9 +478,8 @@ func fromChatRequest(r ChatCompletionRequest) (*api.ChatRequest, error) {
options["stop"] = stops
}
if r.ContextWindow != nil {
slog.Info("context_window in if", "context_window", *r.ContextWindow)
options["num_ctx"] = *r.ContextWindow
if r.ContextLength != nil {
options["num_ctx"] = *r.ContextLength
}
// Deprecated: MaxTokens is deprecated, use MaxCompletionTokens instead

View File

@ -315,11 +315,11 @@ func TestChatMiddleware(t *testing.T) {
},
},
{
name: "chat handler with context_window",
name: "chat handler with context_length",
body: `{
"model": "test-model",
"messages": [{"role": "user", "content": "Hello"}],
"context_window": 4096
"context_length": 4096
}`,
req: api.ChatRequest{
Model: "test-model",