input
This commit is contained in:
parent
eb7cc2d1ce
commit
23d5beeb9c
3
llm/ext_server/server.cpp
vendored
3
llm/ext_server/server.cpp
vendored
@ -1855,6 +1855,8 @@ struct llama_server_context
|
|||||||
|
|
||||||
if (batch.n_tokens == 0)
|
if (batch.n_tokens == 0)
|
||||||
{
|
{
|
||||||
|
// HANGING HERE
|
||||||
|
LOG_INFO("no tokens to process", {});
|
||||||
all_slots_are_idle = true;
|
all_slots_are_idle = true;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@ -3200,7 +3202,6 @@ int main(int argc, char **argv) {
|
|||||||
else {
|
else {
|
||||||
image_data = "";
|
image_data = "";
|
||||||
}
|
}
|
||||||
// TODO: prompt needs to represent the image data
|
|
||||||
|
|
||||||
// create and queue the task
|
// create and queue the task
|
||||||
json responses;
|
json responses;
|
||||||
|
@ -265,38 +265,42 @@ func (s *Server) EmbedHandler(c *gin.Context) {
|
|||||||
truncate = false
|
truncate = false
|
||||||
}
|
}
|
||||||
|
|
||||||
inputCheck := true
|
|
||||||
|
|
||||||
if req.Images != nil {
|
|
||||||
inputCheck = false
|
|
||||||
}
|
|
||||||
|
|
||||||
var input []string
|
var input []string
|
||||||
|
|
||||||
if inputCheck {
|
images := make([]llm.ImageData, len(req.Images))
|
||||||
|
for i := range req.Images {
|
||||||
|
images[i] = llm.ImageData{ID: 0, Data: req.Images[i]}
|
||||||
|
}
|
||||||
|
|
||||||
switch i := req.Input.(type) {
|
if req.Images != nil {
|
||||||
case string:
|
// for _, _ := range images {
|
||||||
if len(i) > 0 {
|
// input = append(input, fmt.Sprintf("[img-%d]", i.ID))
|
||||||
input = append(input, i)
|
input = append(input, "[img-0]")
|
||||||
}
|
// }
|
||||||
case []any:
|
req.Input = ""
|
||||||
for _, v := range i {
|
}
|
||||||
if _, ok := v.(string); !ok {
|
|
||||||
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
input = append(input, v.(string))
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(input) == 0 {
|
switch i := req.Input.(type) {
|
||||||
c.JSON(http.StatusOK, api.EmbedResponse{Model: req.Model, Embeddings: [][]float32{}})
|
case string:
|
||||||
return
|
if len(i) > 0 {
|
||||||
|
input = append(input, i)
|
||||||
}
|
}
|
||||||
|
case []any:
|
||||||
|
for _, v := range i {
|
||||||
|
if _, ok := v.(string); !ok {
|
||||||
|
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
input = append(input, v.(string))
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(input) == 0 {
|
||||||
|
c.JSON(http.StatusOK, api.EmbedResponse{Model: req.Model, Embeddings: [][]float32{}})
|
||||||
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
r, m, opts, err := s.scheduleRunner(c.Request.Context(), req.Model, []Capability{}, req.Options, req.KeepAlive)
|
r, m, opts, err := s.scheduleRunner(c.Request.Context(), req.Model, []Capability{}, req.Options, req.KeepAlive)
|
||||||
@ -336,11 +340,6 @@ func (s *Server) EmbedHandler(c *gin.Context) {
|
|||||||
input[i] = s
|
input[i] = s
|
||||||
}
|
}
|
||||||
|
|
||||||
images := make([]llm.ImageData, len(req.Images))
|
|
||||||
for i := range req.Images {
|
|
||||||
images[i] = llm.ImageData{ID: i, Data: req.Images[i]}
|
|
||||||
}
|
|
||||||
|
|
||||||
embeddings, err := r.Embed(c.Request.Context(), input, images)
|
embeddings, err := r.Embed(c.Request.Context(), input, images)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
Loading…
x
Reference in New Issue
Block a user