refactor modelfile parser

parser/parser_test.go

@@ -1,13 +1,16 @@
 package parser
 
 import (
+	"bytes"
+	"fmt"
+	"io"
 	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func Test_Parser(t *testing.T) {
+func TestParser(t *testing.T) {
 
 	input := `
 FROM model1
@@ -35,7 +38,7 @@ TEMPLATE template1
 	assert.Equal(t, expectedCommands, commands)
 }
 
-func Test_Parser_NoFromLine(t *testing.T) {
+func TestParserNoFromLine(t *testing.T) {
 
 	input := `
 PARAMETER param1 value1
@@ -48,7 +51,7 @@ PARAMETER param2 value2
 	assert.ErrorContains(t, err, "no FROM line")
 }
 
-func Test_Parser_MissingValue(t *testing.T) {
+func TestParserParametersMissingValue(t *testing.T) {
 
 	input := `
 FROM foo
@@ -58,41 +61,292 @@ PARAMETER param1
 	reader := strings.NewReader(input)
 
 	_, err := Parse(reader)
-	assert.ErrorContains(t, err, "missing value for [param1]")
-
+	assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
 }
 
-func Test_Parser_Messages(t *testing.T) {
-
-	input := `
+func TestParserMessages(t *testing.T) {
+	var cases = []struct {
+		input    string
+		expected []Command
+		err      error
+	}{
+		{
+			`
+FROM foo
+MESSAGE system You are a Parser. Always Parse things.
+`,
+			[]Command{
+				{Name: "model", Args: "foo"},
+				{Name: "message", Args: "system: You are a Parser. Always Parse things."},
+			},
+			nil,
+		},
+		{
+			`
 FROM foo
 MESSAGE system You are a Parser. Always Parse things.
 MESSAGE user Hey there!
 MESSAGE assistant Hello, I want to parse all the things!
-`
-
-	reader := strings.NewReader(input)
-	commands, err := Parse(reader)
-	assert.Nil(t, err)
-
-	expectedCommands := []Command{
-		{Name: "model", Args: "foo"},
-		{Name: "message", Args: "system: You are a Parser. Always Parse things."},
-		{Name: "message", Args: "user: Hey there!"},
-		{Name: "message", Args: "assistant: Hello, I want to parse all the things!"},
-	}
-
-	assert.Equal(t, expectedCommands, commands)
-}
-
-func Test_Parser_Messages_BadRole(t *testing.T) {
-
-	input := `
+`,
+			[]Command{
+				{Name: "model", Args: "foo"},
+				{Name: "message", Args: "system: You are a Parser. Always Parse things."},
+				{Name: "message", Args: "user: Hey there!"},
+				{Name: "message", Args: "assistant: Hello, I want to parse all the things!"},
+			},
+			nil,
+		},
+		{
+			`
+FROM foo
+MESSAGE system """
+You are a multiline Parser. Always Parse things.
+"""
+			`,
+			[]Command{
+				{Name: "model", Args: "foo"},
+				{Name: "message", Args: "system: \nYou are a multiline Parser. Always Parse things.\n"},
+			},
+			nil,
+		},
+		{
+			`
 FROM foo
 MESSAGE badguy I'm a bad guy!
-`
+`,
+			nil,
+			errInvalidRole,
+		},
+		{
+			`
+FROM foo
+MESSAGE system
+`,
+			nil,
+			io.ErrUnexpectedEOF,
+		},
+		{
+			`
+FROM foo
+MESSAGE system`,
+			nil,
+			io.ErrUnexpectedEOF,
+		},
+	}
 
-	reader := strings.NewReader(input)
-	_, err := Parse(reader)
-	assert.ErrorContains(t, err, "role must be one of \"system\", \"user\", or \"assistant\"")
+	for _, c := range cases {
+		t.Run("", func(t *testing.T) {
+			commands, err := Parse(strings.NewReader(c.input))
+			assert.ErrorIs(t, err, c.err)
+			assert.Equal(t, c.expected, commands)
+		})
+	}
+}
+
+func TestParserQuoted(t *testing.T) {
+	var cases = []struct {
+		multiline string
+		expected  []Command
+		err       error
+	}{
+		{
+			`
+FROM foo
+TEMPLATE """
+This is a
+multiline template.
+"""
+			`,
+			[]Command{
+				{Name: "model", Args: "foo"},
+				{Name: "template", Args: "\nThis is a\nmultiline template.\n"},
+			},
+			nil,
+		},
+		{
+			`
+FROM foo
+TEMPLATE """
+This is a
+multiline template."""
+			`,
+			[]Command{
+				{Name: "model", Args: "foo"},
+				{Name: "template", Args: "\nThis is a\nmultiline template."},
+			},
+			nil,
+		},
+		{
+			`
+FROM foo
+TEMPLATE """This is a
+multiline template."""
+			`,
+			[]Command{
+				{Name: "model", Args: "foo"},
+				{Name: "template", Args: "This is a\nmultiline template."},
+			},
+			nil,
+		},
+		{
+			`
+FROM foo
+TEMPLATE """This is a multiline template."""
+			`,
+			[]Command{
+				{Name: "model", Args: "foo"},
+				{Name: "template", Args: "This is a multiline template."},
+			},
+			nil,
+		},
+		{
+			`
+FROM foo
+TEMPLATE """This is a multiline template.""
+			`,
+			nil,
+			io.ErrUnexpectedEOF,
+		},
+		{
+			`
+FROM foo
+TEMPLATE "
+			`,
+			nil,
+			io.ErrUnexpectedEOF,
+		},
+		{
+			`
+FROM foo
+TEMPLATE """
+This is a multiline template with "quotes".
+"""
+`,
+			[]Command{
+				{Name: "model", Args: "foo"},
+				{Name: "template", Args: "\nThis is a multiline template with \"quotes\".\n"},
+			},
+			nil,
+		},
+		{
+			`
+FROM foo
+TEMPLATE """"""
+`,
+			[]Command{
+				{Name: "model", Args: "foo"},
+				{Name: "template", Args: ""},
+			},
+			nil,
+		},
+		{
+			`
+FROM foo
+TEMPLATE ""
+`,
+			[]Command{
+				{Name: "model", Args: "foo"},
+				{Name: "template", Args: ""},
+			},
+			nil,
+		},
+		{
+			`
+FROM foo
+TEMPLATE "'"
+`,
+			[]Command{
+				{Name: "model", Args: "foo"},
+				{Name: "template", Args: "'"},
+			},
+			nil,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run("", func(t *testing.T) {
+			commands, err := Parse(strings.NewReader(c.multiline))
+			assert.ErrorIs(t, err, c.err)
+			assert.Equal(t, c.expected, commands)
+		})
+	}
+}
+
+func TestParserParameters(t *testing.T) {
+	var cases = []string{
+		"numa true",
+		"num_ctx 1",
+		"num_batch 1",
+		"num_gqa 1",
+		"num_gpu 1",
+		"main_gpu 1",
+		"low_vram true",
+		"f16_kv true",
+		"logits_all true",
+		"vocab_only true",
+		"use_mmap true",
+		"use_mlock true",
+		"num_thread 1",
+		"num_keep 1",
+		"seed 1",
+		"num_predict 1",
+		"top_k 1",
+		"top_p 1.0",
+		"tfs_z 1.0",
+		"typical_p 1.0",
+		"repeat_last_n 1",
+		"temperature 1.0",
+		"repeat_penalty 1.0",
+		"presence_penalty 1.0",
+		"frequency_penalty 1.0",
+		"mirostat 1",
+		"mirostat_tau 1.0",
+		"mirostat_eta 1.0",
+		"penalize_newline true",
+		"stop foo",
+	}
+
+	for _, c := range cases {
+		t.Run(c, func(t *testing.T) {
+			var b bytes.Buffer
+			fmt.Fprintln(&b, "FROM foo")
+			fmt.Fprintln(&b, "PARAMETER", c)
+			t.Logf("input: %s", b.String())
+			_, err := Parse(&b)
+			assert.Nil(t, err)
+		})
+	}
+}
+
+func TestParserOnlyFrom(t *testing.T) {
+	commands, err := Parse(strings.NewReader("FROM foo"))
+	assert.Nil(t, err)
+
+	expected := []Command{{Name: "model", Args: "foo"}}
+	assert.Equal(t, expected, commands)
+}
+
+func TestParserComments(t *testing.T) {
+	var cases = []struct {
+		input    string
+		expected []Command
+	}{
+		{
+			`
+# comment
+FROM foo
+	`,
+			[]Command{
+				{Name: "model", Args: "foo"},
+			},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run("", func(t *testing.T) {
+			commands, err := Parse(strings.NewReader(c.input))
+			assert.Nil(t, err)
+			assert.Equal(t, c.expected, commands)
+		})
+	}
 }