model: support tools streaming and improve parsing

server/model.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"slices"
 	"strings"
+	gotmpl "text/template"
 	"text/template/parse"
 
 	"github.com/ollama/ollama/api"
@@ -129,33 +130,122 @@ func detectContentType(r io.Reader) (string, error) {
 	return "unknown", nil
 }
 
-func parseObjects(s string) []map[string]any {
+// extractToolCallsTemplate finds the immediate following text after any IfNode containing ".ToolCalls"
-	var objs []map[string]any
+func extractToolCallsTemplate(tmpl *gotmpl.Template) (string, bool) {
-	for offset := 0; offset < len(s); {
+	if tmpl == nil || tmpl.Tree == nil {
-		var obj map[string]any
+		slog.Debug("TextAfterToolCalls: template or tree is nil")
-		decoder := json.NewDecoder(strings.NewReader(s[offset:]))
+		return "", false
-		if err := decoder.Decode(&obj); errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
+	}
-			break
+
-		} else if syntax := &(json.SyntaxError{}); errors.As(err, &syntax) {
+	var result string
-			// skip over any syntax errors
+	var found bool
-			offset += int(syntax.Offset)
+
-		} else if unmarshalType := &(json.UnmarshalTypeError{}); errors.As(err, &unmarshalType) {
+	var walk func(nodes []parse.Node)
-			// skip over any unmarshalable types
+	walk = func(nodes []parse.Node) {
-			offset += int(unmarshalType.Offset)
+		for _, node := range nodes {
-		} else if err != nil {
+			if found {
-			return nil
+				return
-		} else {
+			}
-			offset += int(decoder.InputOffset())
+
-			objs = append(objs, obj)
+			switch n := node.(type) {
+			case *parse.IfNode:
+				if nodeContainsToolCalls(n) {
+					// Collect immediate TextNode(s) at start of IfNode's list
+					var sb strings.Builder
+					for _, innerNode := range n.List.Nodes {
+						if tn, ok := innerNode.(*parse.TextNode); ok {
+							sb.Write(tn.Text)
+						} else {
+							// Stop at first non-text node
+							break
+						}
+					}
+					result = sb.String()
+					found = true
+					return
+				}
+				// Recurse into child nodes
+				walk(n.List.Nodes)
+				if n.ElseList != nil {
+					walk(n.ElseList.Nodes)
+				}
+			case *parse.ListNode:
+				walk(n.Nodes)
+			case *parse.RangeNode:
+				walk(n.List.Nodes)
+				if n.ElseList != nil {
+					walk(n.ElseList.Nodes)
+				}
+			case *parse.WithNode:
+				walk(n.List.Nodes)
+				if n.ElseList != nil {
+					walk(n.ElseList.Nodes)
+				}
+			default:
+				// Continue to next node
+				continue
+			}
+
+			if found {
+				return
+			}
 		}
 	}
 
-	return objs
+	walk(tmpl.Tree.Root.Nodes)
+	return result, found
 }
 
-// parseToolCalls attempts to parse a JSON string into a slice of ToolCalls.
+// Helper to detect if a node's condition includes ".ToolCalls"
-// mxyng: this only really works if the input contains tool calls in some JSON format
+func nodeContainsToolCalls(n *parse.IfNode) bool {
-func (m *Model) parseToolCalls(s string) ([]api.ToolCall, bool) {
+	for _, cmd := range n.Pipe.Cmds {
+		for _, arg := range cmd.Args {
+			if field, ok := arg.(*parse.FieldNode); ok {
+				if slices.Contains(field.Ident, "ToolCalls") {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
+func ToolToken(tmpl *gotmpl.Template) (string, bool) {
+	tokenText, ok := extractToolCallsTemplate(tmpl)
+	if !ok {
+		return "", false
+	}
+	tokenText = strings.TrimSpace(tokenText)
+	if tokenText == "" {
+		return "", false
+	}
+	first := strings.Fields(tokenText)[0]
+
+	start := -1
+	end := -1
+	for i, r := range tokenText {
+		if r == '<' || r == '[' {
+			start = i
+		}
+		if (r == '>' || r == ']') && start != -1 {
+			end = i
+			break
+		}
+	}
+	if start != -1 && end != -1 {
+		// return the token including the [ or < and the ] or >
+		return tokenText[start : end+1], true
+	} else if start != -1 {
+		// get until the [ or < - in the case tag was not closed
+		return tokenText[:start], true
+	} else if end != -1 {
+		// get after the ] or > - in the case tag was not opened
+		return tokenText[end+1:], true
+	}
+	return first, true
+}
+
+func ToolTemplate(m *Model) (*gotmpl.Template, bool) {
 	// create a subtree from the node that ranges over .ToolCalls
 	tmpl := m.Template.Subtree(func(n parse.Node) bool {
 		if t, ok := n.(*parse.RangeNode); ok {
@@ -169,83 +259,5 @@ func (m *Model) parseToolCalls(s string) ([]api.ToolCall, bool) {
 		return nil, false
 	}
 
-	var b bytes.Buffer
+	return tmpl, true
-	if err := tmpl.Execute(&b, map[string][]api.ToolCall{
-		"ToolCalls": {
-			{
-				Function: api.ToolCallFunction{
-					Name: "@@name@@",
-					Arguments: api.ToolCallFunctionArguments{
-						"@@argument@@": 1,
-					},
-				},
-			},
-		},
-	}); err != nil {
-		return nil, false
-	}
-
-	templateObjects := parseObjects(b.String())
-	if len(templateObjects) == 0 {
-		return nil, false
-	}
-
-	// find the keys that correspond to the name and arguments fields
-	var name, arguments string
-	for k, v := range templateObjects[0] {
-		switch v.(type) {
-		case string:
-			name = k
-		case map[string]any:
-			arguments = k
-		}
-	}
-
-	if name == "" || arguments == "" {
-		return nil, false
-	}
-
-	responseObjects := parseObjects(s)
-	if len(responseObjects) == 0 {
-		return nil, false
-	}
-
-	// collect all nested objects
-	var collect func(any) []map[string]any
-	collect = func(obj any) (all []map[string]any) {
-		switch o := obj.(type) {
-		case map[string]any:
-			all = append(all, o)
-			for _, v := range o {
-				all = append(all, collect(v)...)
-			}
-		case []any:
-			for _, v := range o {
-				all = append(all, collect(v)...)
-			}
-		}
-
-		return all
-	}
-
-	var objs []map[string]any
-	for _, p := range responseObjects {
-		objs = append(objs, collect(p)...)
-	}
-
-	var toolCalls []api.ToolCall
-	for _, kv := range objs {
-		n, nok := kv[name].(string)
-		a, aok := kv[arguments].(map[string]any)
-		if nok && aok {
-			toolCalls = append(toolCalls, api.ToolCall{
-				Function: api.ToolCallFunction{
-					Name:      n,
-					Arguments: a,
-				},
-			})
-		}
-	}
-
-	return toolCalls, len(toolCalls) > 0
 }

server/model_test.go

@@ -1,178 +1,184 @@
 package server
 
 import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"os"
-	"path/filepath"
 	"testing"
-
+	gotmpl "text/template"
-	"github.com/google/go-cmp/cmp"
-
-	"github.com/ollama/ollama/api"
-	"github.com/ollama/ollama/template"
 )
 
-func readFile(t *testing.T, base, name string) *bytes.Buffer {
+func TestToolToken(t *testing.T) {
-	t.Helper()
-
-	bts, err := os.ReadFile(filepath.Join(base, name))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	return bytes.NewBuffer(bts)
-}
-
-func TestExecuteWithTools(t *testing.T) {
-	p := filepath.Join("testdata", "tools")
 	cases := []struct {
-		model  string
+		name     string
-		output string
+		template string
-		ok     bool
+		want     string
+		ok       bool
 	}{
-		{"mistral", `[TOOL_CALLS]  [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, true},
-		{"mistral", `[TOOL_CALLS]  [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]
-
-The temperature in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.`, true},
-		{"mistral", `[TOOL_CALLS]  [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"To }]`, false},
-		{"mistral", `I'm not aware of that information. However, I can suggest searching for the weather using the "get_current_weather" function:
-
-		[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, true},
-		{"mistral", " The weather in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.", false},
-		{"command-r-plus", "Action: ```json" + `
-[
-    {
-        "tool_name": "get_current_weather",
-        "parameters": {
-            "format": "fahrenheit",
-            "location": "San Francisco, CA"
-        }
-    },
-    {
-        "tool_name": "get_current_weather",
-        "parameters": {
-            "format": "celsius",
-            "location": "Toronto, Canada"
-        }
-    }
-]
-` + "```", true},
-		{"command-r-plus", " The weather in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.", false},
-		{"firefunction", ` functools[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, true},
-		{"firefunction", " The weather in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.", false},
-		{"llama3-groq-tool-use", `<tool_call>
-{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}}
-{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}
-</tool_call>`, true},
-		{"xlam", `{"tool_calls": [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]}`, true},
-		{"nemotron", `<toolcall>{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]} </toolcall>`, true},
-	}
-
-	var tools []api.Tool
-	if err := json.Unmarshal(readFile(t, p, "tools.json").Bytes(), &tools); err != nil {
-		t.Fatal(err)
-	}
-
-	var messages []api.Message
-	if err := json.Unmarshal(readFile(t, p, "messages.json").Bytes(), &messages); err != nil {
-		t.Fatal(err)
-	}
-
-	calls := []api.ToolCall{
 		{
-			Function: api.ToolCallFunction{
+			name:     "basic tool call with action prefix",
-				Name: "get_current_weather",
+			template: "{{if .ToolCalls}}Action: ```json{{end}}",
-				Arguments: api.ToolCallFunctionArguments{
+			want:     "Action:",
-					"format":   "fahrenheit",
+			ok:       true,
-					"location": "San Francisco, CA",
-				},
-			},
 		},
 		{
-			Function: api.ToolCallFunction{
+			name:     "incomplete functools bracket",
-				Name: "get_current_weather",
+			template: "{{if .ToolCalls}}functools[{{end}}",
-				Arguments: api.ToolCallFunctionArguments{
+			want:     "functools",
-					"format":   "celsius",
+			ok:       true,
-					"location": "Toronto, Canada",
+		},
-				},
+		{
-			},
+			name:     "tool call with angle brackets",
+			template: "{{if .ToolCalls}}Hello, world! <tool_call>{{end}}",
+			want:     "<tool_call>",
+			ok:       true,
+		},
+		{
+			name:     "multiple tool call formats",
+			template: "{{if .ToolCalls}}[tool_call] <tool_call>{{end}}",
+			want:     "[tool_call]",
+			ok:       true,
+		},
+		{
+			name:     "single angle bracket tool call",
+			template: "{{if .ToolCalls}}<tool_call>{{end}}",
+			want:     "<tool_call>",
+			ok:       true,
+		},
+		{
+			name:     "incomplete angle bracket after tool call",
+			template: "{{if .ToolCalls}}[tool_call] <{{end}}",
+			want:     "[tool_call]",
+			ok:       true,
+		},
+		{
+			name:     "angle bracket prefix with tool call",
+			template: "{{if .ToolCalls}}> <tool_call>{{end}}",
+			want:     "<tool_call>",
+			ok:       true,
+		},
+		{
+			name:     "uppercase tool call with incomplete bracket",
+			template: "{{if .ToolCalls}}[TOOL_CALL] [{{end}}",
+			want:     "[TOOL_CALL]",
+			ok:       true,
+		},
+		{
+			name:     "uppercase tool call with adjacent bracket",
+			template: "{{if .ToolCalls}}[TOOL_CALL][{{end}}",
+			want:     "[TOOL_CALL]",
+			ok:       true,
+		},
+		{
+			name:     "tool call with pipe delimiters",
+			template: "{{if .ToolCalls}}<|tool_call|>{{end}}",
+			want:     "<|tool_call|>",
+			ok:       true,
 		},
 	}
 
 	for _, tt := range cases {
-		t.Run(tt.model, func(t *testing.T) {
+		t.Run(tt.name, func(t *testing.T) {
-			tmpl, err := template.Parse(readFile(t, p, fmt.Sprintf("%s.gotmpl", tt.model)).String())
+			tmpl, err := gotmpl.New("test").Parse(tt.template)
 			if err != nil {
-				t.Fatal(err)
+				t.Fatalf("failed to parse template: %v", err)
+			}
+			got, ok := ToolToken(tmpl)
+			if got != tt.want {
+				t.Errorf("ToolToken(%q) = %q; want %q", tt.template, got, tt.want)
+			}
+			if ok != tt.ok {
+				t.Errorf("ToolToken(%q) = %v; want %v", tt.template, ok, tt.ok)
 			}
-
-			t.Run("template", func(t *testing.T) {
-				var actual bytes.Buffer
-				if err := tmpl.Execute(&actual, template.Values{Tools: tools, Messages: messages}); err != nil {
-					t.Fatal(err)
-				}
-
-				if diff := cmp.Diff(actual.String(), readFile(t, p, fmt.Sprintf("%s.out", tt.model)).String()); diff != "" {
-					t.Errorf("mismatch (-got +want):\n%s", diff)
-				}
-			})
-
-			t.Run("parse", func(t *testing.T) {
-				m := &Model{Template: tmpl}
-				actual, ok := m.parseToolCalls(tt.output)
-				if ok != tt.ok {
-					t.Fatalf("expected %t, got %t", tt.ok, ok)
-				}
-
-				if tt.ok {
-					if diff := cmp.Diff(actual, calls); diff != "" {
-						t.Errorf("mismatch (-got +want):\n%s", diff)
-					}
-				}
-			})
 		})
 	}
 }
 
-func TestParseObjects(t *testing.T) {
+func TestTextAfterToolCalls(t *testing.T) {
-	tests := []struct {
+	cases := []struct {
-		input string
+		name     string
-		want  []map[string]any
+		template string
+		want     string
+		ok       bool
 	}{
 		{
-			input: `[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`,
+			name:     "basic tool call with text after",
-			want: []map[string]any{
+			template: `{{if .ToolCalls}}tool response{{end}}`,
-				{"name": "get_current_weather", "arguments": map[string]any{"format": "fahrenheit", "location": "San Francisco, CA"}},
+			want:     "tool response",
-				{"name": "get_current_weather", "arguments": map[string]any{"format": "celsius", "location": "Toronto, Canada"}},
+			ok:       true,
-			},
 		},
 		{
-			input: `<toolcall>{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} </toolcall>`,
+			name:     "tool call with mixed content after",
-			want: []map[string]any{
+			template: `{{if .ToolCalls}}<tool_call>{{.Something}}{{end}}`,
-				{"name": "get_current_weather", "arguments": map[string]any{"format": "fahrenheit", "location": "San Francisco, CA"}},
+			want:     "<tool_call>",
-			},
+			ok:       true,
 		},
 		{
-			input: `<toolcall>{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} </toolcall> <toolcall>{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, ON"}} </toolcall>`,
+			name:     "tool call with no text after",
-			want: []map[string]any{
+			template: `{{if .ToolCalls}}{{.Something}}{{end}}`,
-				{"name": "get_current_weather", "arguments": map[string]any{"format": "fahrenheit", "location": "San Francisco, CA"}},
+			want:     "",
-				{"name": "get_current_weather", "arguments": map[string]any{"format": "celsius", "location": "Toronto, ON"}},
+			ok:       true,
-			},
 		},
 		{
-			input: `{"name": "get_current_weather", "arguments": `,
+			name:     "nested tool call",
-			want:  nil,
+			template: `{{if .Something}}{{if .ToolCalls}}[TOOL_CALL]{{end}}{{end}}`,
+			want:     "[TOOL_CALL]",
+			ok:       true,
+		},
+		{
+			name:     "no tool calls",
+			template: `{{if .Something}}no tools here{{end}}`,
+			want:     "",
+			ok:       false,
+		},
+		{
+			name:     "empty template",
+			template: ``,
+			want:     "",
+			ok:       false,
+		},
+		{
+			name:     "multiple tool calls sections",
+			template: `{{if .ToolCalls}}first{{end}}{{if .ToolCalls}}second{{end}}`,
+			want:     "first",
+			ok:       true,
+		},
+		{
+			name:     "range over tool calls",
+			template: `{{if .ToolCalls}}{{range .ToolCalls}}tool{{end}}{{end}}`,
+			want:     "",
+			ok:       true,
+		},
+		{
+			name:     "tool calls with pipe delimiters",
+			template: `{{if .ToolCalls}}<|tool|>{{end}}`,
+			want:     "<|tool|>",
+			ok:       true,
+		},
+		{
+			name:     "tool calls with nested template",
+			template: `{{if .ToolCalls}}{{template "tool" .}}{{end}}`,
+			want:     "",
+			ok:       true,
+		},
+		{
+			name:     "tool calls with whitespace variations",
+			template: `{{if .ToolCalls}}  tool  {{end}}`,
+			want:     "  tool  ",
+			ok:       true,
 		},
 	}
 
-	for _, tc := range tests {
+	for _, tt := range cases {
-		t.Run(tc.input, func(t *testing.T) {
+		t.Run(tt.name, func(t *testing.T) {
-			got := parseObjects(tc.input)
+			tmpl, err := gotmpl.New("test").Parse(tt.template)
+			if err != nil {
+				t.Fatalf("failed to parse template: %v", err)
+			}
 
-			if diff := cmp.Diff(got, tc.want); diff != "" {
+			got, ok := extractToolCallsTemplate(tmpl)
-				t.Errorf("mismatch (-got +want):\n%s", diff)
+			if got != tt.want {
+				t.Errorf("TextAfterToolCalls() got = %q, want %q", got, tt.want)
+			}
+			if ok != tt.ok {
+				t.Errorf("TextAfterToolCalls() ok = %v, want %v", ok, tt.ok)
 			}
 		})
 	}

server/routes.go

@@ -21,6 +21,7 @@ import (
 	"slices"
 	"strings"
 	"syscall"
+	gotmpl "text/template"
 	"time"
 
 	"github.com/gin-contrib/cors"
@@ -1487,6 +1488,24 @@ func (s *Server) ChatHandler(c *gin.Context) {
 		defer close(ch)
 		var sb strings.Builder
 		var toolCallIndex int = 0
+		var templateToolToken string
+		var tmpl *gotmpl.Template
+		if len(req.Tools) > 0 {
+			var ok bool
+			templateToolToken, ok = ToolToken(m.Template.Template)
+			if !ok {
+				slog.Debug("no tool token found")
+			}
+			tmpl, ok = ToolTemplate(m)
+			if !ok {
+				slog.Debug("no tool template found")
+			}
+		}
+
+		checkToolCall := false
+		if len(req.Tools) > 0 {
+			checkToolCall = true
+		}
 		if err := r.Completion(c.Request.Context(), llm.CompletionRequest{
 			Prompt:  prompt,
 			Images:  images,
@@ -1507,42 +1526,50 @@ func (s *Server) ChatHandler(c *gin.Context) {
 			}
 
 			if r.Done {
+				if sb.Len() > 0 {
+					res.Message.Content = sb.String()
+				}
 				res.DoneReason = r.DoneReason.String()
 				res.TotalDuration = time.Since(checkpointStart)
 				res.LoadDuration = checkpointLoaded.Sub(checkpointStart)
 			}
 
-			// TODO: tool call checking and filtering should be moved outside of this callback once streaming
-			// however this was a simple change for now without reworking streaming logic of this (and other)
-			// handlers
-			if req.Stream != nil && !*req.Stream || len(req.Tools) == 0 {
-				ch <- res
-				return
-			}
-
-			// Streaming tool calls:
-			// If tools are recognized, use a flag to track the sending of a tool downstream
-			// This ensures that content is cleared from the message on the last chunk sent
 			sb.WriteString(r.Content)
-			if toolCalls, ok := m.parseToolCalls(sb.String()); ok {
+			if len(req.Tools) > 0 && checkToolCall {
-				res.Message.ToolCalls = toolCalls
+				slog.Debug("parse tool calls", "content", sb.String(), "templateToolToken", templateToolToken)
-				for i := range toolCalls {
+				toolCalls, partial, err := ParseToolCalls(sb.String(), templateToolToken, tmpl)
-					toolCalls[i].Function.Index = toolCallIndex
+				if err == nil {
-					toolCallIndex++
+					if partial {
+						// circuit break to remove tool end token
+						if len(toolCalls) > 0 {
+							sb.Reset()
+						}
+						// If the tool call is partial, we need to wait for the next chunk
+						return
+					}
+					res.Message.ToolCalls = toolCalls
+					for i := range toolCalls {
+						toolCalls[i].Function.Index = toolCallIndex
+						toolCallIndex++
+					}
+					res.Message.Content = ""
+					sb.Reset()
+					ch <- res
+					// Only way to have multiple calls is to have [] which is derived or provided
+					if templateToolToken == "" {
+						checkToolCall = false
+					}
+					return
 				}
-				res.Message.Content = ""
-				sb.Reset()
-				ch <- res
-				return
 			}
 
-			if r.Done {
+			// If there is no template tool token, we don't need to check for tool calls after the first chunk
-				// Send any remaining content if no tool calls were detected
+			if templateToolToken == "" {
-				if toolCallIndex == 0 {
+				checkToolCall = false
-					res.Message.Content = sb.String()
-				}
-				ch <- res
 			}
+			res.Message.Content = sb.String()
+			sb.Reset()
+			ch <- res
 		}); err != nil {
 			ch <- gin.H{"error": err.Error()}
 		}
@@ -1551,11 +1578,15 @@ func (s *Server) ChatHandler(c *gin.Context) {
 	if req.Stream != nil && !*req.Stream {
 		var resp api.ChatResponse
 		var sb strings.Builder
+		var toolCalls []api.ToolCall
 		for rr := range ch {
 			switch t := rr.(type) {
 			case api.ChatResponse:
 				sb.WriteString(t.Message.Content)
 				resp = t
+				if len(req.Tools) > 0 {
+					toolCalls = append(toolCalls, t.Message.ToolCalls...)
+				}
 			case gin.H:
 				msg, ok := t["error"].(string)
 				if !ok {
@@ -1571,12 +1602,8 @@ func (s *Server) ChatHandler(c *gin.Context) {
 		}
 
 		resp.Message.Content = sb.String()
-
+		if len(toolCalls) > 0 {
-		if len(req.Tools) > 0 {
+			resp.Message.ToolCalls = toolCalls
-			if toolCalls, ok := m.parseToolCalls(sb.String()); ok {
-				resp.Message.ToolCalls = toolCalls
-				resp.Message.Content = ""
-			}
 		}
 
 		c.JSON(http.StatusOK, resp)

server/testdata/tools/qwen2.5-coder.gotmpl

@@ -0,0 +1,51 @@
+{{- if .Suffix }}<|fim_prefix|>{{ .Prompt }}<|fim_suffix|>{{ .Suffix }}<|fim_middle|>
+{{- else if .Messages }}
+{{- if or .System .Tools }}<|im_start|>system
+{{- if .System }}
+{{ .System }}
+{{- end }}
+{{- if .Tools }}
+
+# Tools
+
+You may call one or more functions to assist with the user query.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{{- range .Tools }}
+{"type": "function", "function": {{ .Function }}}
+{{- end }}
+</tools>
+
+For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
+<tool_call>
+{"name": <function-name>, "arguments": <args-json-object>}
+</tool_call>
+{{- end }}<|im_end|>
+{{ end }}
+{{- range $i, $_ := .Messages }}
+{{- $last := eq (len (slice $.Messages $i)) 1 -}}
+{{- if eq .Role "user" }}<|im_start|>user
+{{ .Content }}<|im_end|>
+{{ else if eq .Role "assistant" }}<|im_start|>assistant
+{{ if .Content }}{{ .Content }}
+{{- else if .ToolCalls }}<tool_call>
+{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}}
+{{ end }}</tool_call>
+{{- end }}{{ if not $last }}<|im_end|>
+{{ end }}
+{{- else if eq .Role "tool" }}<|im_start|>user
+<tool_response>
+{{ .Content }}
+</tool_response><|im_end|>
+{{ end }}
+{{- if and (ne .Role "assistant") $last }}<|im_start|>assistant
+{{ end }}
+{{- end }}
+{{- else }}
+{{- if .System }}<|im_start|>system
+{{ .System }}<|im_end|>
+{{ end }}{{ if .Prompt }}<|im_start|>user
+{{ .Prompt }}<|im_end|>
+{{ end }}<|im_start|>assistant
+{{ end }}{{ .Response }}{{ if .Response }}<|im_end|>{{ end }}

server/testdata/tools/qwen2.5-coder.out

@@ -0,0 +1,31 @@
+<|im_start|>system
+You are a knowledgeable assistant. You can answer questions and perform tasks.
+
+# Tools
+
+You may call one or more functions to assist with the user query.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{"type": "function", "function": {"name":"get_current_weather","description":"Get the current weather","parameters":{"type":"object","required":["location","format"],"properties":{"format":{"type":"string","description":"The temperature unit to use. Infer this from the user's location.","enum":["celsius","fahrenheit"]},"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"}}}}}
+</tools>
+
+For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
+<tool_call>
+{"name": <function-name>, "arguments": <args-json-object>}
+</tool_call><|im_end|>
+<|im_start|>user
+What's the weather like today in Paris?<|im_end|>
+<|im_start|>assistant
+<tool_call>
+{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Paris, France"}}
+</tool_call><|im_end|>
+<|im_start|>user
+<tool_response>
+22
+</tool_response><|im_end|>
+<|im_start|>assistant
+The current temperature in Paris, France is 22 degrees Celsius.<|im_end|>
+<|im_start|>user
+What's the weather like today in San Francisco and Toronto?<|im_end|>
+<|im_start|>assistant

server/tools.go

@@ -0,0 +1,175 @@
+package server
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	gotmpl "text/template"
+
+	"github.com/ollama/ollama/api"
+)
+
+func parseObjects(s string) []map[string]any {
+	var objs []map[string]any
+	for offset := 0; offset < len(s); {
+		var obj map[string]any
+		decoder := json.NewDecoder(strings.NewReader(s[offset:]))
+		err := decoder.Decode(&obj)
+		switch {
+		case errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF):
+			return objs
+		case err != nil:
+			var syntax *json.SyntaxError
+			var unmarshalType *json.UnmarshalTypeError
+			switch {
+			case errors.As(err, &syntax):
+				offset += int(syntax.Offset)
+				continue
+			case errors.As(err, &unmarshalType):
+				offset += int(unmarshalType.Offset)
+				continue
+			default:
+				return nil
+			}
+		}
+		offset += int(decoder.InputOffset())
+		objs = append(objs, obj)
+	}
+	return objs
+}
+
+// parseJSONToolCalls attempts to parse a JSON string into a slice of ToolCalls.
+// Returns parsed tool calls and a boolean indicating if the JSON is incomplete
+func parseJSONToolCalls(tmpl *gotmpl.Template, s string) ([]api.ToolCall, bool) {
+	var b bytes.Buffer
+	if err := tmpl.Execute(&b, map[string][]api.ToolCall{
+		"ToolCalls": {
+			{
+				Function: api.ToolCallFunction{
+					Name: "@@name@@",
+					Arguments: api.ToolCallFunctionArguments{
+						"@@argument@@": 1,
+					},
+				},
+			},
+		},
+	}); err != nil {
+		return nil, false
+	}
+
+	templateObjects := parseObjects(b.String())
+	if len(templateObjects) == 0 {
+		return nil, false
+	}
+
+	// find the keys that correspond to the name and arguments fields
+	var name, arguments string
+	for k, v := range templateObjects[0] {
+		switch v.(type) {
+		case string:
+			name = k
+		case map[string]any:
+			arguments = k
+		}
+	}
+
+	if name == "" || arguments == "" {
+		return nil, false
+	}
+
+	responseObjects := parseObjects(s)
+	if len(responseObjects) == 0 {
+		return nil, false
+	}
+
+	// collect all nested objects
+	var collect func(any) []map[string]any
+	collect = func(obj any) (all []map[string]any) {
+		switch o := obj.(type) {
+		case map[string]any:
+			all = append(all, o)
+			for _, v := range o {
+				all = append(all, collect(v)...)
+			}
+		case []any:
+			for _, v := range o {
+				all = append(all, collect(v)...)
+			}
+		}
+
+		return all
+	}
+
+	var objs []map[string]any
+	for _, p := range responseObjects {
+		objs = append(objs, collect(p)...)
+	}
+
+	var toolCalls []api.ToolCall
+	for _, kv := range objs {
+		n, nok := kv[name].(string)
+		a, aok := kv[arguments].(map[string]any)
+		if nok && aok {
+			toolCalls = append(toolCalls, api.ToolCall{
+				Function: api.ToolCallFunction{
+					Name:      n,
+					Arguments: a,
+				},
+			})
+		}
+	}
+
+	return toolCalls, len(toolCalls) > 0
+}
+
+// routeToolParsing is a helper function that routes what kind of tool parsing to use
+func routeToolParsing(s string, tmpl *gotmpl.Template) ([]api.ToolCall, bool, bool) {
+	if strings.HasPrefix(s, "[{") || strings.HasPrefix(s, "```") || strings.HasPrefix(s, "{") {
+		if toolCalls, ok := parseJSONToolCalls(tmpl, s); ok {
+			return toolCalls, false, true
+		}
+		// in the case the JSON never finishes, the acuumulated content should be sent downstream
+		return nil, true, true
+	}
+	// TODO(parthsareen): add python tool call support
+	return nil, false, false
+}
+
+// ParseToolCalls extracts tool calls from a string using a tool token prefix or direct JSON parsing.
+// Returns tool calls, whether parsing is incomplete, and any errors.
+func ParseToolCalls(s string, toolToken string, tmpl *gotmpl.Template) ([]api.ToolCall, bool, error) {
+	if tmpl == nil {
+		return nil, false, fmt.Errorf("no template provided")
+	}
+	s = strings.TrimSpace(s)
+	if len(s) == 0 {
+		return nil, false, fmt.Errorf("empty input string")
+	}
+	if toolToken != "" {
+		if strings.HasPrefix(s, toolToken) {
+			s = strings.TrimSpace(s[len(toolToken):])
+			tc, _, ok := routeToolParsing(s, tmpl)
+			if len(tc) == 0 || !ok {
+				return nil, true, nil
+			}
+			return tc, false, nil
+			// Special token end case
+		} else if strings.HasSuffix(s, toolToken[2:]) {
+			tc := api.ToolCall{
+				Function: api.ToolCallFunction{
+					Name: toolToken,
+				},
+			}
+			return []api.ToolCall{tc}, true, nil
+		}
+	}
+
+	tc, partial, ok := routeToolParsing(s, tmpl)
+	if !ok {
+		return nil, false, fmt.Errorf("failed to parse tool calls for input: %q", s)
+	}
+	return tc, partial, nil
+}

server/tools_test.go

@@ -0,0 +1,281 @@
+package server
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+
+	"github.com/ollama/ollama/api"
+	"github.com/ollama/ollama/template"
+)
+
+func readFile(t *testing.T, base, name string) *bytes.Buffer {
+	t.Helper()
+
+	bts, err := os.ReadFile(filepath.Join(base, name))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return bytes.NewBuffer(bts)
+}
+
+func TestParseToolCalls(t *testing.T) {
+	p := filepath.Join("testdata", "tools")
+	t1 := api.ToolCall{
+		Function: api.ToolCallFunction{
+			Name: "get_current_weather",
+			Arguments: api.ToolCallFunctionArguments{
+				"format":   "fahrenheit",
+				"location": "San Francisco, CA",
+			},
+		},
+	}
+	t2 := api.ToolCall{
+		Function: api.ToolCallFunction{
+			Name: "get_current_weather",
+			Arguments: api.ToolCallFunctionArguments{
+				"format":   "celsius",
+				"location": "Toronto, Canada",
+			},
+		},
+	}
+
+	cases := []struct {
+		name     string
+		model    string
+		output   string
+		token    string
+		expected []api.ToolCall
+		wantErr  bool
+	}{
+		{
+			name:     "mistral invalid json",
+			model:    "mistral",
+			output:   `[TOOL_CALLS]  [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_curren}]`,
+			token:    "[TOOL_CALLS]",
+			expected: []api.ToolCall{},
+			wantErr:  true,
+		},
+		{
+			name:     "mistral valid json",
+			model:    "mistral",
+			output:   `[TOOL_CALLS]  [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`,
+			token:    "[TOOL_CALLS]",
+			expected: []api.ToolCall{t1, t2},
+			wantErr:  false,
+		},
+		{
+			name:     "mistral incomplete json",
+			model:    "mistral",
+			output:   `[TOOL_CALLS]  [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, `,
+			token:    "[TOOL_CALLS]",
+			expected: []api.ToolCall{},
+			wantErr:  true,
+		},
+		{
+			name:  "mistral without tool token",
+			model: "mistral",
+			output: `I'm not aware of that information. However, I can suggest searching for the weather using the "get_current_weather" function:
+
+		[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`,
+			token:    "[TOOL_CALLS]",
+			expected: []api.ToolCall{},
+			wantErr:  true,
+		},
+		{
+			name:     "mistral without tool token - tool first",
+			model:    "mistral",
+			output:   `[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`,
+			token:    "[TOOL_CALLS]",
+			expected: []api.ToolCall{t1, t2},
+			wantErr:  false,
+		},
+		{
+			name:  "command-r-plus with json block",
+			model: "command-r-plus",
+			output: "Action: ```json" + `
+		[
+		    {
+		        "tool_name": "get_current_weather",
+		        "parameters": {
+		            "format": "fahrenheit",
+		            "location": "San Francisco, CA"
+		        }
+		    },
+		    {
+		        "tool_name": "get_current_weather",
+		        "parameters": {
+		            "format": "celsius",
+		            "location": "Toronto, Canada"
+		        }
+		    }
+		]
+		` + "```",
+			token:    "Action:",
+			expected: []api.ToolCall{t1, t2},
+			wantErr:  false,
+		},
+		{
+			name:     "firefunction with functools",
+			model:    "firefunction",
+			output:   ` functools[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`,
+			token:    "functools",
+			expected: []api.ToolCall{t1, t2},
+			wantErr:  false,
+		},
+		{
+			name:  "llama3 with tool call tags",
+			model: "llama3-groq-tool-use",
+			output: `<tool_call>
+		{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}}
+		</tool_call>`,
+			token:    "<tool_call>",
+			expected: []api.ToolCall{t1},
+			wantErr:  false,
+		},
+		{
+			name:     "xlam with tool_calls wrapper",
+			model:    "xlam",
+			output:   `{"tool_calls": [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]}`,
+			token:    "",
+			expected: []api.ToolCall{t1, t2},
+			wantErr:  false,
+		},
+		{
+			name:     "qwen with single tool call",
+			model:    "qwen2.5-coder",
+			output:   `<tool_call>{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}}</tool_call>`,
+			token:    "<tool_call>",
+			expected: []api.ToolCall{t1},
+			wantErr:  false,
+		},
+		{
+			name:     "qwen with invalid tool token",
+			model:    "qwen2.5-coder",
+			output:   `[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}}, {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`,
+			token:    "[TOOL_CALLS]",
+			expected: []api.ToolCall{t1, t2},
+			wantErr:  false,
+		},
+		{
+			name:     "qwen with no tool calls",
+			model:    "qwen2.5-coder",
+			output:   " The weather in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.",
+			token:    "",
+			expected: []api.ToolCall{},
+			wantErr:  true,
+		},
+	}
+
+	var tools []api.Tool
+	if err := json.Unmarshal(readFile(t, p, "tools.json").Bytes(), &tools); err != nil {
+		t.Fatal(err)
+	}
+
+	var messages []api.Message
+	if err := json.Unmarshal(readFile(t, p, "messages.json").Bytes(), &messages); err != nil {
+		t.Fatal(err)
+	}
+
+	for _, tt := range cases {
+		t.Run(tt.model, func(t *testing.T) {
+			tmpl, err := template.Parse(readFile(t, p, fmt.Sprintf("%s.gotmpl", tt.model)).String())
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			t.Run("template", func(t *testing.T) {
+				var actual bytes.Buffer
+				if err := tmpl.Execute(&actual, template.Values{Tools: tools, Messages: messages}); err != nil {
+					t.Fatal(err)
+				}
+
+				if diff := cmp.Diff(actual.String(), readFile(t, p, fmt.Sprintf("%s.out", tt.model)).String()); diff != "" {
+					t.Errorf("mismatch (-got +want):\n%s", diff)
+				}
+			})
+
+			t.Run("parse", func(t *testing.T) {
+				m := &Model{Template: tmpl}
+				tmpl, ok := ToolTemplate(m)
+				if !ok {
+					t.Fatal("no tool template found")
+				}
+				got := []api.ToolCall{}
+				tokens := strings.Fields(tt.output)
+				sb := strings.Builder{}
+				success := false
+				for _, tok := range tokens {
+					sb.WriteString(" " + tok)
+					toolCalls, partial, err := ParseToolCalls(sb.String(), tt.token, tmpl)
+					if err == nil {
+						success = true
+					}
+					if partial {
+						continue
+					}
+					got = append(got, toolCalls...)
+					sb.Reset()
+				}
+
+				if !tt.wantErr {
+					if diff := cmp.Diff(got, tt.expected); diff != "" {
+						t.Errorf("mismatch (-got +want):\n%s", diff)
+					}
+				}
+				if !success && !tt.wantErr {
+					t.Errorf("expected success but got errors")
+				}
+			})
+		})
+	}
+}
+
+func TestParseObjects(t *testing.T) {
+	tests := []struct {
+		input string
+		want  []map[string]any
+	}{
+		{
+			input: `[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`,
+			want: []map[string]any{
+				{"name": "get_current_weather", "arguments": map[string]any{"format": "fahrenheit", "location": "San Francisco, CA"}},
+				{"name": "get_current_weather", "arguments": map[string]any{"format": "celsius", "location": "Toronto, Canada"}},
+			},
+		},
+		{
+			input: `<some_token>{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} </toolcall>`,
+			want: []map[string]any{
+				{"name": "get_current_weather", "arguments": map[string]any{"format": "fahrenheit", "location": "San Francisco, CA"}},
+			},
+		},
+		{
+			input: `<some_token>{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} </toolcall> <toolcall>{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, ON"}} </toolcall>`,
+			want: []map[string]any{
+				{"name": "get_current_weather", "arguments": map[string]any{"format": "fahrenheit", "location": "San Francisco, CA"}},
+				{"name": "get_current_weather", "arguments": map[string]any{"format": "celsius", "location": "Toronto, ON"}},
+			},
+		},
+		{
+			input: `{"name": "get_current_weather", "arguments": `,
+			want:  nil,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.input, func(t *testing.T) {
+			got := parseObjects(tc.input)
+
+			if diff := cmp.Diff(got, tc.want); diff != "" {
+				t.Errorf("mismatch (-got +want):\n%s", diff)
+			}
+		})
+	}
+}