change name to context_length

This commit is contained in:
ParthSareen 2025-02-07 11:48:22 -08:00
parent 61a5254115
commit b4de2e9189
3 changed files with 13 additions and 14 deletions

View File

@ -206,29 +206,29 @@ curl http://localhost:11434/v1/embeddings \
## Extra arguments ## Extra arguments
### Setting context window size ### Setting context length
- `context_window` parameter can be used to set the context window for the model - `context_length` parameter can be used to set the context length for the model
#### OpenAI python library #### OpenAI python library
- OpenAI python library does not support setting context window size, however this can be set for Ollama through the `extra_body` parameter - OpenAI python library does not support setting context length, however this can be set for Ollama through the `extra_body` parameter
```py ```py
completion = client.chat.completions.create( completion = client.chat.completions.create(
model="llama3.1:8b", model="llama3.1:8b",
messages=[{"role": "user", "content": "Say this is a test"}], messages=[{"role": "user", "content": "Say this is a test"}],
extra_body={"context_window": 4096}, extra_body={"context_length": 4096},
) )
``` ```
#### OpenAI JavaScript library #### OpenAI JavaScript library
- OpenAI JavaScript library does not support setting context window size, however this can be set for Ollama by passing `num_ctx` directly with a `@ts-expect-error` as an undocumented parameter in the OpenAI JavaScript library. [See documentation here](https://github.com/openai/openai-node?tab=readme-ov-file#making-customundocumented-requests) - OpenAI JavaScript library does not support setting context length, however this can be set for Ollama by passing `context_length` directly with a `@ts-expect-error` as an undocumented parameter in the OpenAI JavaScript library. [See documentation here](https://github.com/openai/openai-node?tab=readme-ov-file#making-customundocumented-requests)
```ts ```ts
const chatCompletion = await openai.chat.completions.create({ const chatCompletion = await openai.chat.completions.create({
messages: [{ role: 'user', content: 'Say this is a test' }], messages: [{ role: 'user', content: 'Say this is a test' }],
model: 'llama3.2', model: 'llama3.2',
// @ts-expect-error context_window is an additional parameter // @ts-expect-error context_length is an additional parameter
context_window: 4096, context_length: 4096,
}) })
``` ```
@ -239,7 +239,7 @@ curl http://localhost:11434/v1/chat/completions \
-d '{ -d '{
"model": "llama3.2", "model": "llama3.2",
"messages": [{"role": "user", "content": "Say this is a test"}], "messages": [{"role": "user", "content": "Say this is a test"}],
"context_window": 4096 "context_length": 4096
}' }'
``` ```

View File

@ -95,7 +95,7 @@ type ChatCompletionRequest struct {
TopP *float64 `json:"top_p"` TopP *float64 `json:"top_p"`
ResponseFormat *ResponseFormat `json:"response_format"` ResponseFormat *ResponseFormat `json:"response_format"`
Tools []api.Tool `json:"tools"` Tools []api.Tool `json:"tools"`
ContextWindow *int `json:"context_window"` ContextLength *int `json:"context_length"`
} }
type ChatCompletion struct { type ChatCompletion struct {
@ -478,9 +478,8 @@ func fromChatRequest(r ChatCompletionRequest) (*api.ChatRequest, error) {
options["stop"] = stops options["stop"] = stops
} }
if r.ContextWindow != nil { if r.ContextLength != nil {
slog.Info("context_window in if", "context_window", *r.ContextWindow) options["num_ctx"] = *r.ContextLength
options["num_ctx"] = *r.ContextWindow
} }
// Deprecated: MaxTokens is deprecated, use MaxCompletionTokens instead // Deprecated: MaxTokens is deprecated, use MaxCompletionTokens instead

View File

@ -315,11 +315,11 @@ func TestChatMiddleware(t *testing.T) {
}, },
}, },
{ {
name: "chat handler with context_window", name: "chat handler with context_length",
body: `{ body: `{
"model": "test-model", "model": "test-model",
"messages": [{"role": "user", "content": "Hello"}], "messages": [{"role": "user", "content": "Hello"}],
"context_window": 4096 "context_length": 4096
}`, }`,
req: api.ChatRequest{ req: api.ChatRequest{
Model: "test-model", Model: "test-model",