jeans/ollama
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

x/mode: Path -> Name

Browse Source
This commit is contained in:
Blake Mizerany 2024-04-04 14:40:01 -07:00
parent 42cda9dd46
commit 7cd939690a
6 changed files with 93 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
x/build/build.go
View File

@ -52,7 +52,7 @@ func Open(dir string) (*Server, error) {
} }
func (s *Server) Build(ref string, f model.File) error { func (s *Server) Build(ref string, f model.File) error {
mp := model.ParsePath(ref) mp := model.ParseName(ref)
if !mp.CompleteWithoutBuild() { if !mp.CompleteWithoutBuild() {
return fmt.Errorf("%w: %q", ErrIncompleteRef, ref) return fmt.Errorf("%w: %q", ErrIncompleteRef, ref)
} }
@ -112,13 +112,13 @@ func (s *Server) LayerFile(digest string) (string, error) {
} }
func (s *Server) ManifestData(ref string) ([]byte, error) { func (s *Server) ManifestData(ref string) ([]byte, error) {
data, _, err := s.resolve(model.ParsePath(ref)) data, _, err := s.resolve(model.ParseName(ref))
return data, err return data, err
} }
// WeightFile returns the absolute path to the weights file for the given model ref. // WeightFile returns the absolute path to the weights file for the given model ref.
func (s *Server) WeightsFile(ref string) (string, error) { func (s *Server) WeightsFile(ref string) (string, error) {
m, err := s.getManifest(model.ParsePath(ref)) m, err := s.getManifest(model.ParseName(ref))
if err != nil { if err != nil {
return "", err return "", err
} }
@ -139,7 +139,7 @@ func (s *Server) WeightsFile(ref string) (string, error) {
// blob, and then have the ref point to that blob. This would simplify the // blob, and then have the ref point to that blob. This would simplify the
// code, allow us to have integrity checks on the manifest, and clean up // code, allow us to have integrity checks on the manifest, and clean up
// this interface. // this interface.
func (s *Server) resolve(ref model.Path) (data []byte, fileName string, err error) { func (s *Server) resolve(ref model.Name) (data []byte, fileName string, err error) {
fileName, err = s.refFileName(ref) fileName, err = s.refFileName(ref)
if err != nil { if err != nil {
return nil, "", err return nil, "", err
@ -158,11 +158,11 @@ func (s *Server) resolve(ref model.Path) (data []byte, fileName string, err erro
} }
func (s *Server) SetManifestData(ref string, data []byte) error { func (s *Server) SetManifestData(ref string, data []byte) error {
return s.setManifestData(model.ParsePath(ref), data) return s.setManifestData(model.ParseName(ref), data)
} }
// Set sets the data for the given ref. // Set sets the data for the given ref.
func (s *Server) setManifestData(mp model.Path, data []byte) error { func (s *Server) setManifestData(mp model.Name, data []byte) error {
path, err := s.refFileName(mp) path, err := s.refFileName(mp)
if err != nil { if err != nil {
return err return err
@ -176,7 +176,7 @@ func (s *Server) setManifestData(mp model.Path, data []byte) error {
return nil return nil
} }
func (s *Server) refFileName(mp model.Path) (string, error) { func (s *Server) refFileName(mp model.Name) (string, error) {
if !mp.Complete() { if !mp.Complete() {
return "", fmt.Errorf("ref not fully qualified: %q", mp) return "", fmt.Errorf("ref not fully qualified: %q", mp)
} }
@ -196,7 +196,7 @@ type layerJSON struct {
Size int64 `json:"size"` Size int64 `json:"size"`
} }
func (s *Server) getManifest(ref model.Path) (manifestJSON, error) { func (s *Server) getManifest(ref model.Name) (manifestJSON, error) {
data, path, err := s.resolve(ref) data, path, err := s.resolve(ref)
if err != nil { if err != nil {
return manifestJSON{}, err return manifestJSON{}, err

2
x/build/internal/blobstore/store_test.go
View File

@ -68,7 +68,7 @@ func TestStoreBasicBlob(t *testing.T) {
} }
// Check tags // Check tags
name := model.ParsePath("registry.ollama.ai/library/test:latest+KQED") name := model.ParseName("registry.ollama.ai/library/test:latest+KQED")
t.Logf("RESOLVING: %q", name.Parts()) t.Logf("RESOLVING: %q", name.Parts())

2
x/model/file.go
View File

@ -20,7 +20,7 @@ type MessagePragma struct {
type File struct { type File struct {
// From is a required pragma that specifies the source of the model, // From is a required pragma that specifies the source of the model,
// either on disk, or by reference (see blob.ParseRef). // either on disk, or by reference (see model.ParseName).
From string From string
// Optional // Optional

90
x/model/path.go → x/model/name.go
View File

@ -7,35 +7,35 @@ import (
"strings" "strings"
) )
const MaxPathLength = 255 const MaxNameLength = 255
type PathPart int type NamePart int
// Levels of concreteness // Levels of concreteness
const ( const (
Invalid PathPart = iota Invalid NamePart = iota
Domain Registry
Namespace Namespace
Name Short
Tag Tag
Build Build
) )
var kindNames = map[PathPart]string{ var kindNames = map[NamePart]string{
Invalid: "Invalid", Invalid: "Invalid",
Domain: "Domain", Registry: "Domain",
Namespace: "Namespace", Namespace: "Namespace",
Name: "Name", Short: "Name",
Tag: "Tag", Tag: "Tag",
Build: "Build", Build: "Build",
} }
// Path is an opaque reference to a model. // Name is an opaque reference to a model.
// //
// It is comparable and can be used as a map key. // It is comparable and can be used as a map key.
// //
// Users or Path must check Valid before using it. // Users or Name must check Valid before using it.
type Path struct { type Name struct {
domain string domain string
namespace string namespace string
name string name string
@ -46,7 +46,7 @@ type Path struct {
// Format returns a string representation of the ref with the given // Format returns a string representation of the ref with the given
// concreteness. If a part is missing, it is replaced with a loud // concreteness. If a part is missing, it is replaced with a loud
// placeholder. // placeholder.
func (r Path) Full() string { func (r Name) Full() string {
r.domain = cmp.Or(r.domain, "!(MISSING DOMAIN)") r.domain = cmp.Or(r.domain, "!(MISSING DOMAIN)")
r.namespace = cmp.Or(r.namespace, "!(MISSING NAMESPACE)") r.namespace = cmp.Or(r.namespace, "!(MISSING NAMESPACE)")
r.name = cmp.Or(r.name, "!(MISSING NAME)") r.name = cmp.Or(r.name, "!(MISSING NAME)")
@ -55,21 +55,21 @@ func (r Path) Full() string {
return r.String() return r.String()
} }
func (r Path) NameAndTag() string { func (r Name) ShortAndTag() string {
r.domain = "" r.domain = ""
r.namespace = "" r.namespace = ""
r.build = "" r.build = ""
return r.String() return r.String()
} }
func (r Path) NameTagAndBuild() string { func (r Name) ShortTagAndBuild() string {
r.domain = "" r.domain = ""
r.namespace = "" r.namespace = ""
return r.String() return r.String()
} }
// String returns the fully qualified ref string. // String returns the fully qualified ref string.
func (r Path) String() string { func (r Name) String() string {
var b strings.Builder var b strings.Builder
if r.domain != "" { if r.domain != "" {
b.WriteString(r.domain) b.WriteString(r.domain)
@ -93,19 +93,19 @@ func (r Path) String() string {
// Complete reports whether the ref is fully qualified. That is it has a // Complete reports whether the ref is fully qualified. That is it has a
// domain, namespace, name, tag, and build. // domain, namespace, name, tag, and build.
func (r Path) Complete() bool { func (r Name) Complete() bool {
return r.Valid() && !slices.Contains(r.Parts(), "") return r.Valid() && !slices.Contains(r.Parts(), "")
} }
// CompleteWithoutBuild reports whether the ref would be complete if it had a // CompleteWithoutBuild reports whether the ref would be complete if it had a
// valid build. // valid build.
func (r Path) CompleteWithoutBuild() bool { func (r Name) CompleteWithoutBuild() bool {
r.build = "x" r.build = "x"
return r.Valid() && r.Complete() return r.Valid() && r.Complete()
} }
// Less returns true if r is less concrete than o; false otherwise. // Less returns true if r is less concrete than o; false otherwise.
func (r Path) Less(o Path) bool { func (r Name) Less(o Name) bool {
rp := r.Parts() rp := r.Parts()
op := o.Parts() op := o.Parts()
for i := range rp { for i := range rp {
@ -119,7 +119,7 @@ func (r Path) Less(o Path) bool {
// Parts returns the parts of the ref in order of concreteness. // Parts returns the parts of the ref in order of concreteness.
// //
// The length of the returned slice is always 5. // The length of the returned slice is always 5.
func (r Path) Parts() []string { func (r Name) Parts() []string {
return []string{ return []string{
r.domain, r.domain,
r.namespace, r.namespace,
@ -129,13 +129,13 @@ func (r Path) Parts() []string {
} }
} }
func (r Path) Domain() string { return r.namespace } func (r Name) Domain() string { return r.namespace }
func (r Path) Namespace() string { return r.namespace } func (r Name) Namespace() string { return r.namespace }
func (r Path) Name() string { return r.name } func (r Name) Name() string { return r.name }
func (r Path) Tag() string { return r.tag } func (r Name) Tag() string { return r.tag }
func (r Path) Build() string { return r.build } func (r Name) Build() string { return r.build }
// ParsePath parses a model path string into a Path. // ParseName parses a model path string into a Name.
// //
// Examples of valid paths: // Examples of valid paths:
// //
@ -151,26 +151,26 @@ func (r Path) Build() string { return r.build }
// "example.com/mistral:7b+Q4_0+" // "example.com/mistral:7b+Q4_0+"
// "x/y/z/z:8n+I" // "x/y/z/z:8n+I"
// "" // ""
func ParsePath(s string) Path { func ParseName(s string) Name {
var r Path var r Name
for kind, part := range PathParts(s) { for kind, part := range NameParts(s) {
switch kind { switch kind {
case Domain: case Registry:
r.domain = part r.domain = part
case Namespace: case Namespace:
r.namespace = part r.namespace = part
case Name: case Short:
r.name = part r.name = part
case Tag: case Tag:
r.tag = part r.tag = part
case Build: case Build:
r.build = strings.ToUpper(part) r.build = strings.ToUpper(part)
case Invalid: case Invalid:
return Path{} return Name{}
} }
} }
if !r.Valid() { if !r.Valid() {
return Path{} return Name{}
} }
return r return r
} }
@ -179,8 +179,8 @@ func ParsePath(s string) Path {
// The name is left untouched. // The name is left untouched.
// //
// Use this for merging a ref with a default ref. // Use this for merging a ref with a default ref.
func Merge(a, b Path) Path { func Merge(a, b Name) Name {
return Path{ return Name{
// name is left untouched // name is left untouched
name: a.name, name: a.name,
@ -192,7 +192,7 @@ func Merge(a, b Path) Path {
} }
// WithBuild returns a copy of r with the build set to the given string. // WithBuild returns a copy of r with the build set to the given string.
func (r Path) WithBuild(build string) Path { func (r Name) WithBuild(build string) Name {
r.build = build r.build = build
return r return r
} }
@ -202,8 +202,8 @@ func (r Path) WithBuild(build string) Path {
// //
// It normalizes the input string by removing "http://" and "https://" only. // It normalizes the input string by removing "http://" and "https://" only.
// No other normalization is done. // No other normalization is done.
func PathParts(s string) iter.Seq2[PathPart, string] { func NameParts(s string) iter.Seq2[NamePart, string] {
return func(yield func(PathPart, string) bool) { return func(yield func(NamePart, string) bool) {
if strings.HasPrefix(s, "http://") { if strings.HasPrefix(s, "http://") {
s = s[len("http://"):] s = s[len("http://"):]
} }
@ -211,11 +211,11 @@ func PathParts(s string) iter.Seq2[PathPart, string] {
s = s[len("https://"):] s = s[len("https://"):]
} }
if len(s) > MaxPathLength || len(s) == 0 { if len(s) > MaxNameLength || len(s) == 0 {
return return
} }
yieldValid := func(kind PathPart, part string) bool { yieldValid := func(kind NamePart, part string) bool {
if !isValidPart(part) { if !isValidPart(part) {
yield(Invalid, "") yield(Invalid, "")
return false return false
@ -243,15 +243,15 @@ func PathParts(s string) iter.Seq2[PathPart, string] {
if !yieldValid(Tag, s[i+1:j]) { if !yieldValid(Tag, s[i+1:j]) {
return return
} }
state, j = Name, i state, j = Short, i
default: default:
yield(Invalid, "") yield(Invalid, "")
return return
} }
case '/': case '/':
switch state { switch state {
case Name, Tag, Build: case Short, Tag, Build:
if !yieldValid(Name, s[i+1:j]) { if !yieldValid(Short, s[i+1:j]) {
return return
} }
state, j = Namespace, i state, j = Namespace, i
@ -259,7 +259,7 @@ func PathParts(s string) iter.Seq2[PathPart, string] {
if !yieldValid(Namespace, s[i+1:j]) { if !yieldValid(Namespace, s[i+1:j]) {
return return
} }
state, j = Domain, i state, j = Registry, i
default: default:
yield(Invalid, "") yield(Invalid, "")
return return
@ -275,14 +275,14 @@ func PathParts(s string) iter.Seq2[PathPart, string] {
if state <= Namespace { if state <= Namespace {
yieldValid(state, s[:j]) yieldValid(state, s[:j])
} else { } else {
yieldValid(Name, s[:j]) yieldValid(Short, s[:j])
} }
} }
} }
// Valid returns true if the ref has a valid name. To know if a ref is // Valid returns true if the ref has a valid name. To know if a ref is
// "complete", use Complete. // "complete", use Complete.
func (r Path) Valid() bool { func (r Name) Valid() bool {
// Parts ensures we only have valid parts, so no need to validate // Parts ensures we only have valid parts, so no need to validate
// them here, only check if we have a name or not. // them here, only check if we have a name or not.
return r.name != "" return r.name != ""

74
x/model/path_test.go → x/model/name_test.go
View File

@ -6,7 +6,7 @@ import (
"testing" "testing"
) )
var testPaths = map[string]Path{ var testNames = map[string]Name{
"mistral:latest": {name: "mistral", tag: "latest"}, "mistral:latest": {name: "mistral", tag: "latest"},
"mistral": {name: "mistral"}, "mistral": {name: "mistral"},
"mistral:30B": {name: "mistral", tag: "30B"}, "mistral:30B": {name: "mistral", tag: "30B"},
@ -36,33 +36,33 @@ var testPaths = map[string]Path{
"file:///etc/passwd:latest": {}, "file:///etc/passwd:latest": {},
"file:///etc/passwd:latest+u": {}, "file:///etc/passwd:latest+u": {},
strings.Repeat("a", MaxPathLength): {name: strings.Repeat("a", MaxPathLength)}, strings.Repeat("a", MaxNameLength): {name: strings.Repeat("a", MaxNameLength)},
strings.Repeat("a", MaxPathLength+1): {}, strings.Repeat("a", MaxNameLength+1): {},
} }
func TestPathParts(t *testing.T) { func TestNameParts(t *testing.T) {
const wantNumParts = 5 const wantNumParts = 5
var p Path var p Name
if len(p.Parts()) != wantNumParts { if len(p.Parts()) != wantNumParts {
t.Errorf("Parts() = %d; want %d", len(p.Parts()), wantNumParts) t.Errorf("Parts() = %d; want %d", len(p.Parts()), wantNumParts)
} }
} }
func TestParsePath(t *testing.T) { func TestParseName(t *testing.T) {
for s, want := range testPaths { for s, want := range testNames {
for _, prefix := range []string{"", "https://", "http://"} { for _, prefix := range []string{"", "https://", "http://"} {
// We should get the same results with or without the // We should get the same results with or without the
// http(s) prefixes // http(s) prefixes
s := prefix + s s := prefix + s
t.Run(s, func(t *testing.T) { t.Run(s, func(t *testing.T) {
got := ParsePath(s) got := ParseName(s)
if got != want { if got != want {
t.Errorf("ParsePath(%q) = %q; want %q", s, got, want) t.Errorf("ParseName(%q) = %q; want %q", s, got, want)
} }
// test round-trip // test round-trip
if ParsePath(got.String()) != got { if ParseName(got.String()) != got {
t.Errorf("String() = %s; want %s", got.String(), s) t.Errorf("String() = %s; want %s", got.String(), s)
} }
@ -76,7 +76,7 @@ func TestParsePath(t *testing.T) {
} }
} }
func TestPathComplete(t *testing.T) { func TestName(t *testing.T) {
cases := []struct { cases := []struct {
in string in string
complete bool complete bool
@ -92,8 +92,8 @@ func TestPathComplete(t *testing.T) {
for _, tt := range cases { for _, tt := range cases {
t.Run(tt.in, func(t *testing.T) { t.Run(tt.in, func(t *testing.T) {
p := ParsePath(tt.in) p := ParseName(tt.in)
t.Logf("ParsePath(%q) = %#v", tt.in, p) t.Logf("ParseName(%q) = %#v", tt.in, p)
if g := p.Complete(); g != tt.complete { if g := p.Complete(); g != tt.complete {
t.Errorf("Complete(%q) = %v; want %v", tt.in, g, tt.complete) t.Errorf("Complete(%q) = %v; want %v", tt.in, g, tt.complete)
} }
@ -104,7 +104,7 @@ func TestPathComplete(t *testing.T) {
} }
} }
func TestPathStringVariants(t *testing.T) { func TestNameStringVariants(t *testing.T) {
cases := []struct { cases := []struct {
in string in string
nameAndTag string nameAndTag string
@ -116,19 +116,19 @@ func TestPathStringVariants(t *testing.T) {
for _, tt := range cases { for _, tt := range cases {
t.Run(tt.in, func(t *testing.T) { t.Run(tt.in, func(t *testing.T) {
p := ParsePath(tt.in) p := ParseName(tt.in)
t.Logf("ParsePath(%q) = %#v", tt.in, p) t.Logf("ParseName(%q) = %#v", tt.in, p)
if g := p.NameAndTag(); g != tt.nameAndTag { if g := p.ShortAndTag(); g != tt.nameAndTag {
t.Errorf("NameAndTag(%q) = %q; want %q", tt.in, g, tt.nameAndTag) t.Errorf("ShortAndTag(%q) = %q; want %q", tt.in, g, tt.nameAndTag)
} }
if g := p.NameTagAndBuild(); g != tt.nameTagAndBuild { if g := p.ShortTagAndBuild(); g != tt.nameTagAndBuild {
t.Errorf("NameTagAndBuild(%q) = %q; want %q", tt.in, g, tt.nameTagAndBuild) t.Errorf("ShortTagAndBuild(%q) = %q; want %q", tt.in, g, tt.nameTagAndBuild)
} }
}) })
} }
} }
func TestPathFull(t *testing.T) { func TestNameFull(t *testing.T) {
const empty = "!(MISSING DOMAIN)/!(MISSING NAMESPACE)/!(MISSING NAME):!(MISSING TAG)+!(MISSING BUILD)" const empty = "!(MISSING DOMAIN)/!(MISSING NAMESPACE)/!(MISSING NAME):!(MISSING TAG)+!(MISSING BUILD)"
cases := []struct { cases := []struct {
@ -151,8 +151,8 @@ func TestPathFull(t *testing.T) {
for _, tt := range cases { for _, tt := range cases {
t.Run(tt.in, func(t *testing.T) { t.Run(tt.in, func(t *testing.T) {
p := ParsePath(tt.in) p := ParseName(tt.in)
t.Logf("ParsePath(%q) = %#v", tt.in, p) t.Logf("ParseName(%q) = %#v", tt.in, p)
if g := p.Full(); g != tt.wantFull { if g := p.Full(); g != tt.wantFull {
t.Errorf("Full(%q) = %q; want %q", tt.in, g, tt.wantFull) t.Errorf("Full(%q) = %q; want %q", tt.in, g, tt.wantFull)
} }
@ -160,44 +160,44 @@ func TestPathFull(t *testing.T) {
} }
} }
func TestParsePathAllocs(t *testing.T) { func TestParseNameAllocs(t *testing.T) {
// test allocations // test allocations
var r Path var r Name
allocs := testing.AllocsPerRun(1000, func() { allocs := testing.AllocsPerRun(1000, func() {
r = ParsePath("example.com/mistral:7b+Q4_0") r = ParseName("example.com/mistral:7b+Q4_0")
}) })
_ = r _ = r
if allocs > 0 { if allocs > 0 {
t.Errorf("ParsePath allocs = %v; want 0", allocs) t.Errorf("ParseName allocs = %v; want 0", allocs)
} }
} }
func BenchmarkParsePath(b *testing.B) { func BenchmarkParseName(b *testing.B) {
b.ReportAllocs() b.ReportAllocs()
var r Path var r Name
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
r = ParsePath("example.com/mistral:7b+Q4_0") r = ParseName("example.com/mistral:7b+Q4_0")
} }
_ = r _ = r
} }
func FuzzParsePath(f *testing.F) { func FuzzParseName(f *testing.F) {
f.Add("example.com/mistral:7b+Q4_0") f.Add("example.com/mistral:7b+Q4_0")
f.Add("example.com/mistral:7b+q4_0") f.Add("example.com/mistral:7b+q4_0")
f.Add("example.com/mistral:7b+x") f.Add("example.com/mistral:7b+x")
f.Add("x/y/z:8n+I") f.Add("x/y/z:8n+I")
f.Fuzz(func(t *testing.T, s string) { f.Fuzz(func(t *testing.T, s string) {
r0 := ParsePath(s) r0 := ParseName(s)
if !r0.Valid() { if !r0.Valid() {
if r0 != (Path{}) { if r0 != (Name{}) {
t.Errorf("expected invalid path to be zero value; got %#v", r0) t.Errorf("expected invalid path to be zero value; got %#v", r0)
} }
t.Skipf("invalid path: %q", s) t.Skipf("invalid path: %q", s)
} }
for _, p := range r0.Parts() { for _, p := range r0.Parts() {
if len(p) > MaxPathLength { if len(p) > MaxNameLength {
t.Errorf("part too long: %q", p) t.Errorf("part too long: %q", p)
} }
} }
@ -206,7 +206,7 @@ func FuzzParsePath(f *testing.F) {
t.Errorf("String() did not round-trip with case insensitivity: %q\ngot = %q\nwant = %q", s, r0.String(), s) t.Errorf("String() did not round-trip with case insensitivity: %q\ngot = %q\nwant = %q", s, r0.String(), s)
} }
r1 := ParsePath(r0.String()) r1 := ParseName(r0.String())
if r0 != r1 { if r0 != r1 {
t.Errorf("round-trip mismatch: %+v != %+v", r0, r1) t.Errorf("round-trip mismatch: %+v != %+v", r0, r1)
} }
@ -216,8 +216,8 @@ func FuzzParsePath(f *testing.F) {
func ExampleMerge() { func ExampleMerge() {
r := Merge( r := Merge(
ParsePath("mistral"), ParseName("mistral"),
ParsePath("registry.ollama.com/XXXXX:latest+Q4_0"), ParseName("registry.ollama.com/XXXXX:latest+Q4_0"),
) )
fmt.Println(r) fmt.Println(r)

2
x/registry/server.go
View File

@ -82,7 +82,7 @@ func (s *Server) handlePush(w http.ResponseWriter, r *http.Request) error {
return err return err
} }
mp := model.ParsePath(pr.Name) mp := model.ParseName(pr.Name)
if !mp.Complete() { if !mp.Complete() {
return oweb.Invalid("name", pr.Name, "must be complete") return oweb.Invalid("name", pr.Name, "must be complete")
} }