Compare commits
5 Commits
main
...
parth/temp
Author | SHA1 | Date | |
---|---|---|---|
![]() |
6556540655 | ||
![]() |
3f60fd57e3 | ||
![]() |
38cd80d52c | ||
![]() |
c9a46140e6 | ||
![]() |
1d529d8b7b |
10
api/types.go
10
api/types.go
@ -103,10 +103,18 @@ type ChatRequest struct {
|
|||||||
// Tools is an optional list of tools the model has access to.
|
// Tools is an optional list of tools the model has access to.
|
||||||
Tools `json:"tools,omitempty"`
|
Tools `json:"tools,omitempty"`
|
||||||
|
|
||||||
|
Debug *Debug `json:"debug,omitempty"`
|
||||||
|
|
||||||
|
Dry bool `json:"dry,omitempty"`
|
||||||
|
|
||||||
// Options lists model-specific options.
|
// Options lists model-specific options.
|
||||||
Options map[string]interface{} `json:"options"`
|
Options map[string]interface{} `json:"options"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type Debug struct {
|
||||||
|
Include []string `json:"include,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
type Tools []Tool
|
type Tools []Tool
|
||||||
|
|
||||||
func (t Tools) String() string {
|
func (t Tools) String() string {
|
||||||
@ -190,6 +198,8 @@ type ChatResponse struct {
|
|||||||
Message Message `json:"message"`
|
Message Message `json:"message"`
|
||||||
DoneReason string `json:"done_reason,omitempty"`
|
DoneReason string `json:"done_reason,omitempty"`
|
||||||
|
|
||||||
|
Debug map[string]any `json:"debug,omitempty"`
|
||||||
|
|
||||||
Done bool `json:"done"`
|
Done bool `json:"done"`
|
||||||
|
|
||||||
Metrics
|
Metrics
|
||||||
|
@ -82,6 +82,10 @@ func chatPrompt(ctx context.Context, m *Model, tokenize tokenizeFunc, opts *api.
|
|||||||
}
|
}
|
||||||
|
|
||||||
currMsgIdx := n
|
currMsgIdx := n
|
||||||
|
// Warn user if messages are truncated from the input
|
||||||
|
if numTruncatedMessages := len(msgs[0:currMsgIdx]); numTruncatedMessages > 0 {
|
||||||
|
slog.Warn("truncated first messages from input", "num_truncated", numTruncatedMessages)
|
||||||
|
}
|
||||||
|
|
||||||
for cnt, msg := range msgs[currMsgIdx:] {
|
for cnt, msg := range msgs[currMsgIdx:] {
|
||||||
prefix := ""
|
prefix := ""
|
||||||
|
@ -1539,6 +1539,34 @@ func (s *Server) ChatHandler(c *gin.Context) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if req.Dry {
|
||||||
|
var debug map[string]any
|
||||||
|
if req.Debug != nil && req.Debug.Include != nil && slices.Contains(req.Debug.Include, "prompt") {
|
||||||
|
debug = map[string]any{"prompt": prompt}
|
||||||
|
}
|
||||||
|
tokens, err := r.Tokenize(c.Request.Context(), prompt)
|
||||||
|
if err != nil {
|
||||||
|
slog.Error("tokenize error", "error", err)
|
||||||
|
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c.JSON(http.StatusOK, api.ChatResponse{
|
||||||
|
Model: req.Model,
|
||||||
|
CreatedAt: time.Now().UTC(),
|
||||||
|
Message: api.Message{Role: "assistant", Content: ""},
|
||||||
|
Done: true,
|
||||||
|
DoneReason: "dry_run",
|
||||||
|
Debug: debug,
|
||||||
|
Metrics: api.Metrics{
|
||||||
|
PromptEvalCount: len(tokens),
|
||||||
|
PromptEvalDuration: 0,
|
||||||
|
EvalCount: 0,
|
||||||
|
EvalDuration: 0,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
slog.Debug("chat request", "images", len(images), "prompt", prompt)
|
slog.Debug("chat request", "images", len(images), "prompt", prompt)
|
||||||
|
|
||||||
ch := make(chan any)
|
ch := make(chan any)
|
||||||
@ -1571,6 +1599,16 @@ func (s *Server) ChatHandler(c *gin.Context) {
|
|||||||
res.LoadDuration = checkpointLoaded.Sub(checkpointStart)
|
res.LoadDuration = checkpointLoaded.Sub(checkpointStart)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if req.Debug != nil && req.Debug.Include != nil && slices.Contains(req.Debug.Include, "prompt") {
|
||||||
|
res.Debug = map[string]any{"prompt": prompt}
|
||||||
|
if req.Stream != nil && !*req.Stream {
|
||||||
|
tempMsg := res.Message
|
||||||
|
res.Message = api.Message{Role: "assistant", Content: ""}
|
||||||
|
ch <- res
|
||||||
|
res.Message = tempMsg
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TODO: tool call checking and filtering should be moved outside of this callback once streaming
|
// TODO: tool call checking and filtering should be moved outside of this callback once streaming
|
||||||
// however this was a simple change for now without reworking streaming logic of this (and other)
|
// however this was a simple change for now without reworking streaming logic of this (and other)
|
||||||
// handlers
|
// handlers
|
||||||
|
Loading…
x
Reference in New Issue
Block a user