Add dry run option for chat request

This commit is contained in:
ParthSareen 2024-12-19 14:17:29 -08:00
parent c9a46140e6
commit 38cd80d52c
2 changed files with 21 additions and 4 deletions

View File

@ -103,6 +103,10 @@ type ChatRequest struct {
// Tools is an optional list of tools the model has access to.
Tools `json:"tools,omitempty"`
// DryRun when true will prepare and validate the request but stop before sending it to the model.
// This allows inspecting how the prompt would be constructed and what parameters would be used.
DryRun bool `json:"dry_run,omitempty"`
// Options lists model-specific options.
Options map[string]interface{} `json:"options"`
}
@ -185,10 +189,11 @@ func (t *ToolFunction) String() string {
// ChatResponse is the response returned by [Client.Chat]. Its fields are
// similar to [GenerateResponse].
type ChatResponse struct {
Model string `json:"model"`
CreatedAt time.Time `json:"created_at"`
Message Message `json:"message"`
DoneReason string `json:"done_reason,omitempty"`
Model string `json:"model"`
CreatedAt time.Time `json:"created_at"`
Message Message `json:"message"`
DoneReason string `json:"done_reason,omitempty"`
DryRunOutput string `json:"dry_run_output,omitempty"`
Done bool `json:"done"`

View File

@ -1572,6 +1572,18 @@ func (s *Server) ChatHandler(c *gin.Context) {
return
}
if req.DryRun {
c.JSON(http.StatusOK, api.ChatResponse{
Model: req.Model,
CreatedAt: time.Now().UTC(),
Message: api.Message{Role: "assistant", Content: ""},
Done: true,
DoneReason: "dry_run",
DryRunOutput: prompt,
})
return
}
slog.Debug("chat request", "images", len(images), "prompt", prompt)
ch := make(chan any)