jeans/ollama
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

limit num_predict to num_ctx

Browse Source
This commit is contained in:
Jeffrey Morgan 2024-03-12 21:17:25 -07:00
parent 3e22611200
commit ca7c3f7e0f
1 changed files with 13 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
llm/dyn_ext_server.go
View File

@ -172,6 +172,19 @@ func (llm *dynExtServer) Predict(ctx context.Context, predict PredictOpts, fn fu
slog.Info(fmt.Sprintf("loaded %d images", len(predict.Images))) slog.Info(fmt.Sprintf("loaded %d images", len(predict.Images)))
} }
// Limit the number of predictions to the maximum context length
// this will cause no more than two context shifts
// TODO: limit this further to num_ctx - len(prompt) to avoid
// any context shifts at all
if predict.Options.NumPredict > llm.options.NumCtx {
slog.Warn(fmt.Sprintf("requested num_predict is greater than the context length (%d > %d), using %d instead", predict.Options.NumPredict, llm.options.NumCtx, llm.options.NumCtx))
predict.Options.NumPredict = llm.options.NumCtx
}
if predict.Options.NumPredict == -1 {
predict.Options.NumPredict = llm.options.NumCtx
}
request := map[string]any{ request := map[string]any{
"prompt": predict.Prompt, "prompt": predict.Prompt,
"stream": true, "stream": true,