diff --git a/docs/openai.md b/docs/openai.md index 22bb82f78..fddc3b839 100644 --- a/docs/openai.md +++ b/docs/openai.md @@ -206,29 +206,29 @@ curl http://localhost:11434/v1/embeddings \ ## Extra arguments -### Setting context window size -- `context_window` parameter can be used to set the context window for the model +### Setting context length +- `context_length` parameter can be used to set the context length for the model #### OpenAI python library -- OpenAI python library does not support setting context window size, however this can be set for Ollama through the `extra_body` parameter +- OpenAI python library does not support setting context length, however this can be set for Ollama through the `extra_body` parameter ```py completion = client.chat.completions.create( model="llama3.1:8b", messages=[{"role": "user", "content": "Say this is a test"}], - extra_body={"context_window": 4096}, + extra_body={"context_length": 4096}, ) ``` #### OpenAI JavaScript library -- OpenAI JavaScript library does not support setting context window size, however this can be set for Ollama by passing `num_ctx` directly with a `@ts-expect-error` as an undocumented parameter in the OpenAI JavaScript library. [See documentation here](https://github.com/openai/openai-node?tab=readme-ov-file#making-customundocumented-requests) +- OpenAI JavaScript library does not support setting context length, however this can be set for Ollama by passing `context_length` directly with a `@ts-expect-error` as an undocumented parameter in the OpenAI JavaScript library. [See documentation here](https://github.com/openai/openai-node?tab=readme-ov-file#making-customundocumented-requests) ```ts const chatCompletion = await openai.chat.completions.create({ messages: [{ role: 'user', content: 'Say this is a test' }], model: 'llama3.2', - // @ts-expect-error context_window is an additional parameter - context_window: 4096, + // @ts-expect-error context_length is an additional parameter + context_length: 4096, }) ``` @@ -239,7 +239,7 @@ curl http://localhost:11434/v1/chat/completions \ -d '{ "model": "llama3.2", "messages": [{"role": "user", "content": "Say this is a test"}], - "context_window": 4096 + "context_length": 4096 }' ``` diff --git a/openai/openai.go b/openai/openai.go index ebb74c9f6..c91dbd654 100644 --- a/openai/openai.go +++ b/openai/openai.go @@ -95,7 +95,7 @@ type ChatCompletionRequest struct { TopP *float64 `json:"top_p"` ResponseFormat *ResponseFormat `json:"response_format"` Tools []api.Tool `json:"tools"` - ContextWindow *int `json:"context_window"` + ContextLength *int `json:"context_length"` } type ChatCompletion struct { @@ -478,9 +478,8 @@ func fromChatRequest(r ChatCompletionRequest) (*api.ChatRequest, error) { options["stop"] = stops } - if r.ContextWindow != nil { - slog.Info("context_window in if", "context_window", *r.ContextWindow) - options["num_ctx"] = *r.ContextWindow + if r.ContextLength != nil { + options["num_ctx"] = *r.ContextLength } // Deprecated: MaxTokens is deprecated, use MaxCompletionTokens instead diff --git a/openai/openai_test.go b/openai/openai_test.go index 8162db3d1..1302263d7 100644 --- a/openai/openai_test.go +++ b/openai/openai_test.go @@ -315,11 +315,11 @@ func TestChatMiddleware(t *testing.T) { }, }, { - name: "chat handler with context_window", + name: "chat handler with context_length", body: `{ "model": "test-model", "messages": [{"role": "user", "content": "Hello"}], - "context_window": 4096 + "context_length": 4096 }`, req: api.ChatRequest{ Model: "test-model",