jeans/ollama
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

set num_ctx through extra body

Browse Source
This commit is contained in:
ParthSareen 2025-01-29 13:13:11 -08:00
parent 2ef3c803a1
commit 35e97db03b
1 changed files with 32 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
openai/openai.go
View File

@ -84,7 +84,8 @@ type ChatCompletionRequest struct {
Messages []Message `json:"messages"`
Stream bool `json:"stream"`
StreamOptions *StreamOptions `json:"stream_options"`
MaxTokens *int `json:"max_tokens"`
MaxCompletionTokens *int `json:"max_completion_tokens"`
MaxTokens *int `json:"max_tokens" deprecated:"use max_completion_tokens instead"`
Seed *int `json:"seed"`
Stop any `json:"stop"`
Temperature *float64 `json:"temperature"`
@ -93,6 +94,7 @@ type ChatCompletionRequest struct {
TopP *float64 `json:"top_p"`
ResponseFormat *ResponseFormat `json:"response_format"`
Tools []api.Tool `json:"tools"`
NumCtx *int `json:"num_ctx"`
}
type ChatCompletion struct {
@ -475,8 +477,24 @@ func fromChatRequest(r ChatCompletionRequest) (*api.ChatRequest, error) {
options["stop"] = stops
}
// Deprecated: MaxTokens is deprecated, use MaxCompletionTokens instead
if r.MaxTokens != nil {
options["num_predict"] = *r.MaxTokens
r.MaxCompletionTokens = r.MaxTokens
}
if r.NumCtx != nil {
options["num_ctx"] = *r.NumCtx
}
DEFAULT_NUM_CTX := 2048
if r.MaxCompletionTokens != nil {
options["num_predict"] = *r.MaxCompletionTokens
if numCtx, ok := options["num_ctx"].(int); ok && *r.MaxCompletionTokens > numCtx {
options["num_ctx"] = *r.MaxCompletionTokens
} else if *r.MaxCompletionTokens > DEFAULT_NUM_CTX {
options["num_ctx"] = DEFAULT_NUM_CTX
}
}
if r.Temperature != nil {