diff --git a/server/testdata/tools/llama3.2.gotmpl b/server/testdata/tools/llama3.2.gotmpl new file mode 100644 index 000000000..b132423e5 --- /dev/null +++ b/server/testdata/tools/llama3.2.gotmpl @@ -0,0 +1,44 @@ +<|start_header_id|>system<|end_header_id|> + +Cutting Knowledge Date: December 2023 + +{{ if .System }}{{ .System }} +{{- end }} +{{- if .Tools }}When you receive a tool call response, use the output to format an answer to the orginal user question. + +You are a helpful assistant with tool calling capabilities. +{{- end }}<|eot_id|> +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 }} +{{- if eq .Role "user" }}<|start_header_id|>user<|end_header_id|> +{{- if and $.Tools $last }} + +Given the following functions, please respond with a JSON for a function call with its proper arguments that best answers the given prompt. + +Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. Do not use variables. + +{{ range $.Tools }} +{{- . }} +{{ end }} +{{ .Content }}<|eot_id|> +{{- else }} + +{{ .Content }}<|eot_id|> +{{- end }}{{ if $last }}<|start_header_id|>assistant<|end_header_id|> + +{{ end }} +{{- else if eq .Role "assistant" }}<|start_header_id|>assistant<|end_header_id|> +{{- if .ToolCalls }} +{{ range .ToolCalls }} +{"name": "{{ .Function.Name }}", "parameters": {{ .Function.Arguments }}}{{ end }} +{{- else }} + +{{ .Content }} +{{- end }}{{ if not $last }}<|eot_id|>{{ end }} +{{- else if eq .Role "tool" }}<|start_header_id|>ipython<|end_header_id|> + +{{ .Content }}<|eot_id|>{{ if $last }}<|start_header_id|>assistant<|end_header_id|> + +{{ end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/server/testdata/tools/llama3.2.out b/server/testdata/tools/llama3.2.out new file mode 100644 index 000000000..a27c6eafc --- /dev/null +++ b/server/testdata/tools/llama3.2.out @@ -0,0 +1,24 @@ +<|start_header_id|>system<|end_header_id|> + +Cutting Knowledge Date: December 2023 + +You are a knowledgeable assistant. You can answer questions and perform tasks.When you receive a tool call response, use the output to format an answer to the orginal user question. + +You are a helpful assistant with tool calling capabilities.<|eot_id|><|start_header_id|>user<|end_header_id|> + +What's the weather like today in Paris?<|eot_id|><|start_header_id|>assistant<|end_header_id|> + +{"name": "get_current_weather", "parameters": {"format":"celsius","location":"Paris, France"}}<|eot_id|><|start_header_id|>ipython<|end_header_id|> + +22<|eot_id|><|start_header_id|>assistant<|end_header_id|> + +The current temperature in Paris, France is 22 degrees Celsius.<|eot_id|><|start_header_id|>user<|end_header_id|> + +Given the following functions, please respond with a JSON for a function call with its proper arguments that best answers the given prompt. + +Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. Do not use variables. + +{"type":"function","function":{"name":"get_current_weather","description":"Get the current weather","parameters":{"type":"object","required":["location","format"],"properties":{"format":{"type":"string","description":"The temperature unit to use. Infer this from the user's location.","enum":["celsius","fahrenheit"]},"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"}}}}} + +What's the weather like today in San Francisco and Toronto?<|eot_id|><|start_header_id|>assistant<|end_header_id|> + diff --git a/server/testdata/tools/qwen3.gotmpl b/server/testdata/tools/qwen3.gotmpl new file mode 100644 index 000000000..26f6656fa --- /dev/null +++ b/server/testdata/tools/qwen3.gotmpl @@ -0,0 +1,50 @@ +{{- if .Messages }} +{{- if or .System .Tools }}<|im_start|>system +{{- if .System }} +{{ .System }} +{{- end }} +{{- if .Tools }} + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{{- range .Tools }} +{"type": "function", "function": {{ .Function }}} +{{- end }} + + +For each function call, return a json object with function name and arguments within XML tags: + +{"name": , "arguments": } + +{{- end }}<|im_end|> +{{ end }} +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +{{- if eq .Role "user" }}<|im_start|>user +{{ .Content }}<|im_end|> +{{ else if eq .Role "assistant" }}<|im_start|>assistant +{{ if .Content }}{{ .Content }} +{{- else if .ToolCalls }} +{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{ end }} +{{- end }}{{ if not $last }}<|im_end|> +{{ end }} +{{- else if eq .Role "tool" }}<|im_start|>user + +{{ .Content }} +<|im_end|> +{{ end }} +{{- if and (ne .Role "assistant") $last }}<|im_start|>assistant +{{ end }} +{{- end }} +{{- else }} +{{- if .System }}<|im_start|>system +{{ .System }}<|im_end|> +{{ end }}{{ if .Prompt }}<|im_start|>user +{{ .Prompt }}<|im_end|> +{{ end }}<|im_start|>assistant +{{ end }}{{ .Response }}{{ if .Response }}<|im_end|>{{ end }} \ No newline at end of file diff --git a/server/testdata/tools/qwen3.out b/server/testdata/tools/qwen3.out new file mode 100644 index 000000000..76bfbfa98 --- /dev/null +++ b/server/testdata/tools/qwen3.out @@ -0,0 +1,31 @@ +<|im_start|>system +You are a knowledgeable assistant. You can answer questions and perform tasks. + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{"type": "function", "function": {"name":"get_current_weather","description":"Get the current weather","parameters":{"type":"object","required":["location","format"],"properties":{"format":{"type":"string","description":"The temperature unit to use. Infer this from the user's location.","enum":["celsius","fahrenheit"]},"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"}}}}} + + +For each function call, return a json object with function name and arguments within XML tags: + +{"name": , "arguments": } +<|im_end|> +<|im_start|>user +What's the weather like today in Paris?<|im_end|> +<|im_start|>assistant + +{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Paris, France"}} +<|im_end|> +<|im_start|>user + +22 +<|im_end|> +<|im_start|>assistant +The current temperature in Paris, France is 22 degrees Celsius.<|im_end|> +<|im_start|>user +What's the weather like today in San Francisco and Toronto?<|im_end|> +<|im_start|>assistant diff --git a/server/tools.go b/server/tools.go index 248a6b78a..d4401270b 100644 --- a/server/tools.go +++ b/server/tools.go @@ -167,7 +167,7 @@ func (p *ToolParser) parseJSONToolCalls(s string) ([]api.ToolCall, bool, bool) { return toolCalls, false, true } -func (p *ToolParser) updateState(ok bool, partial bool, tcs []api.ToolCall) { +func (p *ToolParser) updateOutputState(ok bool, partial bool, tcs []api.ToolCall) { switch { case !ok && !partial && p.state == ForceTools: fmt.Println("Case: !ok && !partial && ForceTools - staying in force tools, resetting buffer") @@ -188,7 +188,6 @@ func (p *ToolParser) updateState(ok bool, partial bool, tcs []api.ToolCall) { p.sb.Reset() case !ok && partial: fmt.Println("Case: !ok && partial - accumulating partial content") - // ! acucumulate case len(tcs) > 0: @@ -204,68 +203,74 @@ func (p *ToolParser) updateState(ok bool, partial bool, tcs []api.ToolCall) { } } +func (p *ToolParser) updateInputState(s string, hasPrefix bool) (string, bool) { + if p.toolPrefix == "" { + return s, true + } + + if hasPrefix { + p.state = ForceTools + // partial tool possibly + } else if strings.HasPrefix(p.toolPrefix, s) { + slog.Debug("tool prefix partially", "prefix", p.toolPrefix, "content", s) + // TODO: could possibly err maybe this should be greedy instead? + p.state = ForceTools + return "", false + } else if strings.Contains(s, p.toolPrefix) { + idx := strings.Index(s, p.toolPrefix) + if idx != -1 { + // still keeps the prefix + p.state = ContainsPartialPrefix + p.sb.Reset() + p.sb.WriteString(s[idx:]) + return s[:idx], false + } + } + // Special token end case + if strings.HasSuffix(s, p.toolPrefix[2:]) { + // can be with string or just the token + if hasPrefix { + s = strings.TrimSpace(s[:len(s)-(len(p.toolPrefix)+1)]) + } else { + p.state = ToolSuffix + p.sb.Reset() + return "", false + } + slog.Debug("setting to no tool", "content", s) + } + return s, true +} + // ParseToolCalls extracts tool calls from a string using a tool token prefix or direct JSON parsing. // Returns tool calls, whether parsing is incomplete, and any errors. func (p *ToolParser) ParseToolCalls(s string) ([]api.ToolCall, string, bool) { + // append input p.sb.WriteString(s) s = p.sb.String() s = strings.TrimSpace(s) - slog.Debug("parse tool calls", "content", s) if len(s) == 0 { return nil, "", false } - s, hasPrefix := strings.CutPrefix(s, p.toolPrefix) - fmt.Println("hasPrefix", hasPrefix) - var tcs []api.ToolCall - var partial bool - var ok bool - if p.toolPrefix != "" { - if hasPrefix { - p.state = ForceTools - slog.Debug("tool prefix in prefix", "prefix", p.toolPrefix, "content", s) - // partial tool possibly - } else if strings.HasPrefix(p.toolPrefix, s) { - slog.Debug("tool prefix partially", "prefix", p.toolPrefix, "content", s) - // TODO: could possibly err maybe this should be greedy instead? - p.state = ForceTools - return nil, "", false - } else if strings.Contains(s, p.toolPrefix) { - idx := strings.Index(s, p.toolPrefix) - if idx != -1 { - // still keeps the prefix - p.state = ContainsPartialPrefix - p.sb.Reset() - p.sb.WriteString(s[idx:]) - return nil, s[:idx], false - } + s, hasPrefix := strings.CutPrefix(s, p.toolPrefix) + + s, ok := p.updateInputState(s, hasPrefix) + if !ok { + if p.state == ContainsPartialPrefix { + return nil, s, false } - // Special token end case - // if s, ok := strings.CutSuffix(s, p.toolPrefix[2:]); ok { - if strings.HasSuffix(s, p.toolPrefix[2:]) { - // can be with string or just the token - if hasPrefix { - s = strings.TrimSpace(s[:len(s)-(len(p.toolPrefix)+1)]) - } else { - p.state = ToolSuffix - p.sb.Reset() - return nil, "", false - } - slog.Debug("setting to no tool", "content", s) - } - } - fmt.Println("s before parsing", s) - if p.state == SendTokens { - fmt.Println("returning nil cause of send tokens") return nil, "", false } - tcs, partial, ok = p.parseJSONToolCalls(s) - slog.Debug("returning tool calls", "tool calls", tcs) - fmt.Println("end state", p.state) - fmt.Println("len tcs", len(tcs)) - p.updateState(ok, partial, tcs) + if p.state == SendTokens { + return nil, "", false + } + + var tcs []api.ToolCall + var partial bool + tcs, partial, ok = p.parseJSONToolCalls(s) + p.updateOutputState(ok, partial, tcs) if !ok { return nil, "", false } @@ -276,7 +281,6 @@ func (p *ToolParser) ParseToolCalls(s string) ([]api.ToolCall, string, bool) { func NewToolParser(model *Model) *ToolParser { templateToolPrefix, _ := ToolPrefix(model.Template.Template) templateToolPrefix = strings.TrimSpace(templateToolPrefix) - slog.Debug("tool prefix", "prefix", templateToolPrefix) tmpl, ok := ToolTemplate(model) if !ok { return nil @@ -288,6 +292,7 @@ func NewToolParser(model *Model) *ToolParser { } else { state = GreedyToolWithPrefix } + fmt.Println("state", state) return &ToolParser{ tmpl: tmpl, sb: &strings.Builder{}, diff --git a/server/tools_test.go b/server/tools_test.go index ac55c1558..7cffa5c25 100644 --- a/server/tools_test.go +++ b/server/tools_test.go @@ -51,7 +51,6 @@ func TestParseToolCalls(t *testing.T) { name string model string output string - prefix string expected []api.ToolCall wantErr bool }{ @@ -59,7 +58,6 @@ func TestParseToolCalls(t *testing.T) { name: "mistral invalid json", model: "mistral", output: `[TOOL_CALLS] [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_curren}]`, - prefix: "[TOOL_CALLS]", expected: []api.ToolCall{}, wantErr: true, }, @@ -67,7 +65,6 @@ func TestParseToolCalls(t *testing.T) { name: "mistral multiple tool calls - no prefix", model: "mistral", output: `[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, - prefix: "[TOOL_CALLS]", expected: []api.ToolCall{t1, t2}, wantErr: false, }, @@ -76,7 +73,6 @@ func TestParseToolCalls(t *testing.T) { model: "mistral", output: `[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}] model outputs more tokens here and then [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, - prefix: "[TOOL_CALLS]", expected: []api.ToolCall{t1, t2}, wantErr: false, }, @@ -84,7 +80,6 @@ func TestParseToolCalls(t *testing.T) { name: "mistral valid json - with prefix", model: "mistral", output: `[TOOL_CALLS] [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, - prefix: "[TOOL_CALLS]", expected: []api.ToolCall{t1, t2}, wantErr: false, }, @@ -94,7 +89,6 @@ func TestParseToolCalls(t *testing.T) { model: "mistral", output: `[TOOL_CALLS] [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}] model outputs more tokens here and then [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, - prefix: "[TOOL_CALLS]", expected: []api.ToolCall{t1, t2, t1, t2}, wantErr: false, }, @@ -102,7 +96,6 @@ func TestParseToolCalls(t *testing.T) { name: "mistral incomplete json", model: "mistral", output: `[TOOL_CALLS] [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, `, - prefix: "[TOOL_CALLS]", expected: []api.ToolCall{}, wantErr: true, }, @@ -112,7 +105,6 @@ func TestParseToolCalls(t *testing.T) { output: `I'm not aware of that information. However, I can suggest searching for the weather using the "get_current_weather" function: [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, - prefix: "[TOOL_CALLS]", expected: []api.ToolCall{}, wantErr: true, }, @@ -120,7 +112,6 @@ func TestParseToolCalls(t *testing.T) { name: "mistral without tool token - tool first", model: "mistral", output: `[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, - prefix: "[TOOL_CALLS]", expected: []api.ToolCall{t1, t2}, wantErr: false, }, @@ -145,7 +136,6 @@ func TestParseToolCalls(t *testing.T) { } ] ` + "```", - prefix: "Action: ```json", expected: []api.ToolCall{t1, t2}, wantErr: false, }, @@ -153,7 +143,6 @@ func TestParseToolCalls(t *testing.T) { name: "firefunction with functools", model: "firefunction", output: ` functools[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, - prefix: "functools", expected: []api.ToolCall{t1, t2}, wantErr: false, }, @@ -163,7 +152,6 @@ func TestParseToolCalls(t *testing.T) { output: ` {"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} `, - prefix: "", expected: []api.ToolCall{t1}, wantErr: false, }, @@ -171,7 +159,6 @@ func TestParseToolCalls(t *testing.T) { name: "xlam with tool_calls wrapper", model: "xlam", output: `{"tool_calls": [{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}},{"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]}`, - prefix: "", expected: []api.ToolCall{t1, t2}, wantErr: false, }, @@ -179,7 +166,6 @@ func TestParseToolCalls(t *testing.T) { name: "qwen with single tool call", model: "qwen2.5-coder", output: `{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}}`, - prefix: "", expected: []api.ToolCall{t1}, wantErr: false, }, @@ -187,7 +173,6 @@ func TestParseToolCalls(t *testing.T) { name: "qwen with invalid tool token", model: "qwen2.5-coder", output: `[{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}}, {"name": "get_current_weather", "arguments": {"format":"celsius","location":"Toronto, Canada"}}]`, - prefix: "[TOOL_CALLS]", expected: []api.ToolCall{t1, t2}, wantErr: false, }, @@ -195,7 +180,6 @@ func TestParseToolCalls(t *testing.T) { name: "qwen3 with single tool call and thinking", model: "qwen3", output: `Okay, let me think what tool we should use...{"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}}`, - prefix: "", expected: []api.ToolCall{t1}, wantErr: false, }, @@ -203,7 +187,6 @@ func TestParseToolCalls(t *testing.T) { name: "qwen3 with single tool call and thinking spaces", model: "qwen3", output: `Okay, let me think what tool we should use... {"name": "get_current_weather", "arguments": {"format":"fahrenheit","location":"San Francisco, CA"}} `, - prefix: "", expected: []api.ToolCall{t1}, wantErr: false, }, @@ -211,7 +194,20 @@ func TestParseToolCalls(t *testing.T) { name: "qwen with no tool calls", model: "qwen2.5-coder", output: " The weather in San Francisco, CA is 70°F and in Toronto, Canada is 20°C.", - prefix: "", + expected: []api.ToolCall{}, + wantErr: true, + }, + { + name: "llama3.2 with tool call - no prefix", + model: "llama3.2", + output: `{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}}`, + expected: []api.ToolCall{t1}, + wantErr: false, + }, + { + name: "llama3.2 with tool call - in middle", + model: "llama3.2", + output: `some non json text{"name": "get_current_weather", "parameters": {"format":"fahrenheit","location":"San Francisco, CA"}}`, expected: []api.ToolCall{}, wantErr: true, }, @@ -235,11 +231,13 @@ func TestParseToolCalls(t *testing.T) { } t.Run("template", func(t *testing.T) { - var actual bytes.Buffer - if err := tmpl.Execute(&actual, template.Values{Tools: tools, Messages: messages}); err != nil { + actual := &bytes.Buffer{} // Create new buffer for each test + t.Log("template", tmpl, "model", tt.model) + if err := tmpl.Execute(actual, template.Values{Tools: tools, Messages: messages}); err != nil { t.Fatal(err) } + t.Log("actual", actual.String()) if diff := cmp.Diff(actual.String(), readFile(t, p, fmt.Sprintf("%s.out", tt.model)).String()); diff != "" { t.Errorf("mismatch (-got +want):\n%s", diff) }