From 2647a0e4433837673334da88f09d8bfae8571147 Mon Sep 17 00:00:00 2001 From: Roy Han Date: Fri, 26 Jul 2024 15:18:35 -0700 Subject: [PATCH] num parallel embed --- server/sched.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/sched.go b/server/sched.go index 2daed3abb..e1e821072 100644 --- a/server/sched.go +++ b/server/sched.go @@ -132,6 +132,8 @@ func (s *Scheduler) processPending(ctx context.Context) { if len(pending.model.ProjectorPaths) > 0 && numParallel != 1 { numParallel = 1 slog.Warn("multimodal models don't support parallel requests yet") + } else if strings.Contains(pending.model.Config.ModelFamily, "bert") { + numParallel = runtime.NumCPU() } for {