From 23d5beeb9c467696147018840201ae48b1828621 Mon Sep 17 00:00:00 2001 From: Roy Han Date: Tue, 16 Jul 2024 15:19:13 -0700 Subject: [PATCH] input --- llm/ext_server/server.cpp | 3 +- server/routes.go | 63 +++++++++++++++++++-------------------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/llm/ext_server/server.cpp b/llm/ext_server/server.cpp index cec6d4fb0..8b1d736e3 100644 --- a/llm/ext_server/server.cpp +++ b/llm/ext_server/server.cpp @@ -1855,6 +1855,8 @@ struct llama_server_context if (batch.n_tokens == 0) { + // HANGING HERE + LOG_INFO("no tokens to process", {}); all_slots_are_idle = true; return true; } @@ -3200,7 +3202,6 @@ int main(int argc, char **argv) { else { image_data = ""; } - // TODO: prompt needs to represent the image data // create and queue the task json responses; diff --git a/server/routes.go b/server/routes.go index 8e58ddbde..43af26e65 100644 --- a/server/routes.go +++ b/server/routes.go @@ -265,38 +265,42 @@ func (s *Server) EmbedHandler(c *gin.Context) { truncate = false } - inputCheck := true - - if req.Images != nil { - inputCheck = false - } - var input []string - if inputCheck { + images := make([]llm.ImageData, len(req.Images)) + for i := range req.Images { + images[i] = llm.ImageData{ID: 0, Data: req.Images[i]} + } - switch i := req.Input.(type) { - case string: - if len(i) > 0 { - input = append(input, i) - } - case []any: - for _, v := range i { - if _, ok := v.(string); !ok { - c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"}) - return - } - input = append(input, v.(string)) - } - default: - c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"}) - return - } + if req.Images != nil { + // for _, _ := range images { + // input = append(input, fmt.Sprintf("[img-%d]", i.ID)) + input = append(input, "[img-0]") + // } + req.Input = "" + } - if len(input) == 0 { - c.JSON(http.StatusOK, api.EmbedResponse{Model: req.Model, Embeddings: [][]float32{}}) - return + switch i := req.Input.(type) { + case string: + if len(i) > 0 { + input = append(input, i) } + case []any: + for _, v := range i { + if _, ok := v.(string); !ok { + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"}) + return + } + input = append(input, v.(string)) + } + default: + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"}) + return + } + + if len(input) == 0 { + c.JSON(http.StatusOK, api.EmbedResponse{Model: req.Model, Embeddings: [][]float32{}}) + return } r, m, opts, err := s.scheduleRunner(c.Request.Context(), req.Model, []Capability{}, req.Options, req.KeepAlive) @@ -336,11 +340,6 @@ func (s *Server) EmbedHandler(c *gin.Context) { input[i] = s } - images := make([]llm.ImageData, len(req.Images)) - for i := range req.Images { - images[i] = llm.ImageData{ID: i, Data: req.Images[i]} - } - embeddings, err := r.Embed(c.Request.Context(), input, images) if err != nil {