From 17de2b4405a8769aaa64e840a83b1f262e4db49e Mon Sep 17 00:00:00 2001 From: Roy Han Date: Wed, 3 Jul 2024 14:02:25 -0700 Subject: [PATCH] Refactoring of legacy and new --- api/types.go | 2 +- llm/ext_server/server.cpp | 1 - llm/server.go | 60 +-------------------------------------- server/routes.go | 6 ++-- server/sched_test.go | 9 +++--- 5 files changed, 9 insertions(+), 69 deletions(-) diff --git a/api/types.go b/api/types.go index 46748601a..46c0d4178 100644 --- a/api/types.go +++ b/api/types.go @@ -246,7 +246,7 @@ type EmbeddingRequest struct { // EmbeddingResponse is the response from [Client.Embeddings]. type EmbeddingResponse struct { - Embedding []float64 `json:"embedding"` + Embedding []float32 `json:"embedding"` } // CreateRequest is the request passed to [Client.Create]. diff --git a/llm/ext_server/server.cpp b/llm/ext_server/server.cpp index d2a568594..a56e694d9 100644 --- a/llm/ext_server/server.cpp +++ b/llm/ext_server/server.cpp @@ -3191,7 +3191,6 @@ int main(int argc, char **argv) { if (result.result_json.count("results")) { // result for multi-task responses = result.result_json.at("results"); - LOG_INFO("results", {result.result_json}); } else { // result for single task responses = std::vector(1, result.result_json); diff --git a/llm/server.go b/llm/server.go index 6269c5a86..db4431f56 100644 --- a/llm/server.go +++ b/llm/server.go @@ -33,7 +33,7 @@ type LlamaServer interface { Ping(ctx context.Context) error WaitUntilRunning(ctx context.Context) error Completion(ctx context.Context, req CompletionRequest, fn func(CompletionResponse)) error - Embedding(ctx context.Context, prompt string) ([]float64, error) + // Embedding(ctx context.Context, prompt string) ([]float64, error) Embed(ctx context.Context, input []string) ([][]float32, error) Tokenize(ctx context.Context, content string) ([]int, error) Detokenize(ctx context.Context, tokens []int) (string, error) @@ -905,64 +905,6 @@ func (s *llmServer) Embed(ctx context.Context, input []string) ([][]float32, err return embedding.Embedding, nil } -type EmbeddingRequest struct { - Content string `json:"content"` -} - -type EmbeddingResponse struct { - Embedding [][]float64 `json:"embedding"` -} - -func (s *llmServer) Embedding(ctx context.Context, prompt string) ([]float64, error) { - if err := s.sem.Acquire(ctx, 1); err != nil { - slog.Error("Failed to acquire semaphore", "error", err) - return nil, err - } - defer s.sem.Release(1) - - // Make sure the server is ready - status, err := s.getServerStatusRetry(ctx) - if err != nil { - return nil, err - } else if status != ServerStatusReady { - return nil, fmt.Errorf("unexpected server status: %s", status.ToString()) - } - - data, err := json.Marshal(EmbeddingRequest{Content: prompt}) - if err != nil { - return nil, fmt.Errorf("error marshaling embed data: %w", err) - } - - req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/embedding", s.port), bytes.NewBuffer(data)) - if err != nil { - return nil, fmt.Errorf("error creating embed request: %w", err) - } - req.Header.Set("Content-Type", "application/json") - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, fmt.Errorf("do embedding request: %w", err) - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("error reading embed response: %w", err) - } - - if resp.StatusCode >= 400 { - log.Printf("llm encode error: %s", body) - return nil, fmt.Errorf("%s", body) - } - - var embedding EmbeddingResponse - if err := json.Unmarshal(body, &embedding); err != nil { - return nil, fmt.Errorf("unmarshal tokenize response: %w", err) - } - - return embedding.Embedding[0], nil -} - type TokenizeRequest struct { Content string `json:"content"` } diff --git a/server/routes.go b/server/routes.go index 4611ff084..5702aec16 100644 --- a/server/routes.go +++ b/server/routes.go @@ -554,11 +554,11 @@ func (s *Server) EmbeddingsHandler(c *gin.Context) { // an empty request loads the model if req.Prompt == "" { - c.JSON(http.StatusOK, api.EmbeddingResponse{Embedding: []float64{}}) + c.JSON(http.StatusOK, api.EmbeddingResponse{Embedding: []float32{}}) return } - embedding, err := runner.llama.Embedding(c.Request.Context(), req.Prompt) + embedding, err := runner.llama.Embed(c.Request.Context(), []string{req.Prompt}) if err != nil { slog.Info(fmt.Sprintf("embedding generation failed: %v", err)) c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to generate embedding"}) @@ -566,7 +566,7 @@ func (s *Server) EmbeddingsHandler(c *gin.Context) { } resp := api.EmbeddingResponse{ - Embedding: embedding, + Embedding: embedding[0], } c.JSON(http.StatusOK, resp) } diff --git a/server/sched_test.go b/server/sched_test.go index 4fc588b72..e3b1506ba 100644 --- a/server/sched_test.go +++ b/server/sched_test.go @@ -630,8 +630,6 @@ type mockLlm struct { pingResp error waitResp error completionResp error - embeddingResp []float64 - embeddingRespErr error embedResp [][]float32 embedRespErr error tokenizeResp []int @@ -650,9 +648,10 @@ func (s *mockLlm) WaitUntilRunning(ctx context.Context) error { return s.waitRes func (s *mockLlm) Completion(ctx context.Context, req llm.CompletionRequest, fn func(llm.CompletionResponse)) error { return s.completionResp } -func (s *mockLlm) Embedding(ctx context.Context, prompt string) ([]float64, error) { - return s.embeddingResp, s.embeddingRespErr -} + +// func (s *mockLlm) Embedding(ctx context.Context, prompt string) ([]float64, error) { +// return s.embeddingResp, s.embeddingRespErr +// } func (s *mockLlm) Embed(ctx context.Context, input []string) ([][]float32, error) { return s.embedResp, s.embedRespErr }