x/model: move x/build.Ref -> x/model.Path

Also, update names and comments accordingly.
This commit is contained in:
Blake Mizerany 2024-04-04 14:02:31 -07:00
parent de72688b35
commit 58de2b8d4a
11 changed files with 88 additions and 94 deletions

View File

@ -1,4 +1,4 @@
package blob package model
import ( import (
"cmp" "cmp"
@ -7,7 +7,7 @@ import (
"strings" "strings"
) )
const MaxRefLength = 255 const MaxPathLength = 255
type PartKind int type PartKind int
@ -30,12 +30,12 @@ var kindNames = map[PartKind]string{
Build: "Build", Build: "Build",
} }
// Ref is an opaque reference to a blob. // Path is an opaque reference to a model.
// //
// It is comparable and can be used as a map key. // It is comparable and can be used as a map key.
// //
// Users or Ref must check Valid before using it. // Users or Path must check Valid before using it.
type Ref struct { type Path struct {
domain string domain string
namespace string namespace string
name string name string
@ -46,7 +46,7 @@ type Ref struct {
// Format returns a string representation of the ref with the given // Format returns a string representation of the ref with the given
// concreteness. If a part is missing, it is replaced with a loud // concreteness. If a part is missing, it is replaced with a loud
// placeholder. // placeholder.
func (r Ref) Full() string { func (r Path) Full() string {
r.domain = cmp.Or(r.domain, "!(MISSING DOMAIN)") r.domain = cmp.Or(r.domain, "!(MISSING DOMAIN)")
r.namespace = cmp.Or(r.namespace, "!(MISSING NAMESPACE)") r.namespace = cmp.Or(r.namespace, "!(MISSING NAMESPACE)")
r.name = cmp.Or(r.name, "!(MISSING NAME)") r.name = cmp.Or(r.name, "!(MISSING NAME)")
@ -55,21 +55,21 @@ func (r Ref) Full() string {
return r.String() return r.String()
} }
func (r Ref) NameAndTag() string { func (r Path) NameAndTag() string {
r.domain = "" r.domain = ""
r.namespace = "" r.namespace = ""
r.build = "" r.build = ""
return r.String() return r.String()
} }
func (r Ref) NameTagAndBuild() string { func (r Path) NameTagAndBuild() string {
r.domain = "" r.domain = ""
r.namespace = "" r.namespace = ""
return r.String() return r.String()
} }
// String returns the fully qualified ref string. // String returns the fully qualified ref string.
func (r Ref) String() string { func (r Path) String() string {
var b strings.Builder var b strings.Builder
if r.domain != "" { if r.domain != "" {
b.WriteString(r.domain) b.WriteString(r.domain)
@ -93,19 +93,19 @@ func (r Ref) String() string {
// Complete reports whether the ref is fully qualified. That is it has a // Complete reports whether the ref is fully qualified. That is it has a
// domain, namespace, name, tag, and build. // domain, namespace, name, tag, and build.
func (r Ref) Complete() bool { func (r Path) Complete() bool {
return r.Valid() && !slices.Contains(r.Parts(), "") return r.Valid() && !slices.Contains(r.Parts(), "")
} }
// CompleteWithoutBuild reports whether the ref would be complete if it had a // CompleteWithoutBuild reports whether the ref would be complete if it had a
// valid build. // valid build.
func (r Ref) CompleteWithoutBuild() bool { func (r Path) CompleteWithoutBuild() bool {
r.build = "x" r.build = "x"
return r.Valid() && r.Complete() return r.Valid() && r.Complete()
} }
// Less returns true if r is less concrete than o; false otherwise. // Less returns true if r is less concrete than o; false otherwise.
func (r Ref) Less(o Ref) bool { func (r Path) Less(o Path) bool {
rp := r.Parts() rp := r.Parts()
op := o.Parts() op := o.Parts()
for i := range rp { for i := range rp {
@ -119,7 +119,7 @@ func (r Ref) Less(o Ref) bool {
// Parts returns the parts of the ref in order of concreteness. // Parts returns the parts of the ref in order of concreteness.
// //
// The length of the returned slice is always 5. // The length of the returned slice is always 5.
func (r Ref) Parts() []string { func (r Path) Parts() []string {
return []string{ return []string{
r.domain, r.domain,
r.namespace, r.namespace,
@ -129,36 +129,30 @@ func (r Ref) Parts() []string {
} }
} }
func (r Ref) Domain() string { return r.namespace } func (r Path) Domain() string { return r.namespace }
func (r Ref) Namespace() string { return r.namespace } func (r Path) Namespace() string { return r.namespace }
func (r Ref) Name() string { return r.name } func (r Path) Name() string { return r.name }
func (r Ref) Tag() string { return r.tag } func (r Path) Tag() string { return r.tag }
func (r Ref) Build() string { return r.build } func (r Path) Build() string { return r.build }
// ParseRef parses a ref string into a Ref. A ref string is a name, an // ParsePath parses a model path string into a Path.
// optional tag, and an optional build, separated by colons and pluses.
// //
// The name must be valid ascii [a-zA-Z0-9_]. // Examples of valid paths:
// The tag must be valid ascii [a-zA-Z0-9_].
// The build must be valid ascii [a-zA-Z0-9_].
// //
// It returns then zero value if the ref is invalid. // "example.com/mistral:7b+x"
// "example.com/mistral:7b+Q4_0"
// "mistral:7b+x"
// "example.com/x/mistral:latest+Q4_0"
// "example.com/x/mistral:latest"
// //
// // Valid Examples: // Examples of invalid paths:
// ParseRef("mistral:latest") returns ("mistral", "latest", "")
// ParseRef("mistral") returns ("mistral", "", "")
// ParseRef("mistral:30B") returns ("mistral", "30B", "")
// ParseRef("mistral:7b") returns ("mistral", "7b", "")
// ParseRef("mistral:7b+Q4_0") returns ("mistral", "7b", "Q4_0")
// ParseRef("mistral+KQED") returns ("mistral", "latest", "KQED")
// ParseRef(".x.:7b+Q4_0:latest") returns (".x.", "7b", "Q4_0")
// ParseRef("-grok-f.oo:7b+Q4_0") returns ("-grok-f.oo", "7b", "Q4_0")
// //
// // Invalid Examples: // "example.com/mistral:7b+"
// ParseRef("m stral") returns ("", "", "") // zero // "example.com/mistral:7b+Q4_0+"
// ParseRef("... 129 chars ...") returns ("", "", "") // zero // "x/y/z/z:8n+I"
func ParseRef(s string) Ref { // ""
var r Ref func ParsePath(s string) Path {
var r Path
for kind, part := range Parts(s) { for kind, part := range Parts(s) {
switch kind { switch kind {
case Domain: case Domain:
@ -172,11 +166,11 @@ func ParseRef(s string) Ref {
case Build: case Build:
r.build = strings.ToUpper(part) r.build = strings.ToUpper(part)
case Invalid: case Invalid:
return Ref{} return Path{}
} }
} }
if !r.Valid() { if !r.Valid() {
return Ref{} return Path{}
} }
return r return r
} }
@ -185,8 +179,8 @@ func ParseRef(s string) Ref {
// The name is left untouched. // The name is left untouched.
// //
// Use this for merging a ref with a default ref. // Use this for merging a ref with a default ref.
func Merge(a, b Ref) Ref { func Merge(a, b Path) Path {
return Ref{ return Path{
// name is left untouched // name is left untouched
name: a.name, name: a.name,
@ -211,7 +205,7 @@ func Parts(s string) iter.Seq2[PartKind, string] {
s = s[len("https://"):] s = s[len("https://"):]
} }
if len(s) > MaxRefLength || len(s) == 0 { if len(s) > MaxPathLength || len(s) == 0 {
return return
} }
@ -282,7 +276,7 @@ func Parts(s string) iter.Seq2[PartKind, string] {
// Valid returns true if the ref has a valid name. To know if a ref is // Valid returns true if the ref has a valid name. To know if a ref is
// "complete", use Complete. // "complete", use Complete.
func (r Ref) Valid() bool { func (r Path) Valid() bool {
// Parts ensures we only have valid parts, so no need to validate // Parts ensures we only have valid parts, so no need to validate
// them here, only check if we have a name or not. // them here, only check if we have a name or not.
return r.name != "" return r.name != ""

View File

@ -1,4 +1,4 @@
package blob package model
import ( import (
"fmt" "fmt"
@ -6,7 +6,7 @@ import (
"testing" "testing"
) )
var testRefs = map[string]Ref{ var testPaths = map[string]Path{
"mistral:latest": {name: "mistral", tag: "latest"}, "mistral:latest": {name: "mistral", tag: "latest"},
"mistral": {name: "mistral"}, "mistral": {name: "mistral"},
"mistral:30B": {name: "mistral", tag: "30B"}, "mistral:30B": {name: "mistral", tag: "30B"},
@ -36,33 +36,33 @@ var testRefs = map[string]Ref{
"file:///etc/passwd:latest": {}, "file:///etc/passwd:latest": {},
"file:///etc/passwd:latest+u": {}, "file:///etc/passwd:latest+u": {},
strings.Repeat("a", MaxRefLength): {name: strings.Repeat("a", MaxRefLength)}, strings.Repeat("a", MaxPathLength): {name: strings.Repeat("a", MaxPathLength)},
strings.Repeat("a", MaxRefLength+1): {}, strings.Repeat("a", MaxPathLength+1): {},
} }
func TestRefParts(t *testing.T) { func TestPathParts(t *testing.T) {
const wantNumParts = 5 const wantNumParts = 5
var ref Ref var p Path
if len(ref.Parts()) != wantNumParts { if len(p.Parts()) != wantNumParts {
t.Errorf("Parts() = %d; want %d", len(ref.Parts()), wantNumParts) t.Errorf("Parts() = %d; want %d", len(p.Parts()), wantNumParts)
} }
} }
func TestParseRef(t *testing.T) { func TestParsePath(t *testing.T) {
for s, want := range testRefs { for s, want := range testPaths {
for _, prefix := range []string{"", "https://", "http://"} { for _, prefix := range []string{"", "https://", "http://"} {
// We should get the same results with or without the // We should get the same results with or without the
// http(s) prefixes // http(s) prefixes
s := prefix + s s := prefix + s
t.Run(s, func(t *testing.T) { t.Run(s, func(t *testing.T) {
got := ParseRef(s) got := ParsePath(s)
if got != want { if got != want {
t.Errorf("ParseRef(%q) = %q; want %q", s, got, want) t.Errorf("ParsePath(%q) = %q; want %q", s, got, want)
} }
// test round-trip // test round-trip
if ParseRef(got.String()) != got { if ParsePath(got.String()) != got {
t.Errorf("String() = %s; want %s", got.String(), s) t.Errorf("String() = %s; want %s", got.String(), s)
} }
@ -76,7 +76,7 @@ func TestParseRef(t *testing.T) {
} }
} }
func TestRefComplete(t *testing.T) { func TestPathComplete(t *testing.T) {
cases := []struct { cases := []struct {
in string in string
complete bool complete bool
@ -92,19 +92,19 @@ func TestRefComplete(t *testing.T) {
for _, tt := range cases { for _, tt := range cases {
t.Run(tt.in, func(t *testing.T) { t.Run(tt.in, func(t *testing.T) {
ref := ParseRef(tt.in) p := ParsePath(tt.in)
t.Logf("ParseRef(%q) = %#v", tt.in, ref) t.Logf("ParsePath(%q) = %#v", tt.in, p)
if g := ref.Complete(); g != tt.complete { if g := p.Complete(); g != tt.complete {
t.Errorf("Complete(%q) = %v; want %v", tt.in, g, tt.complete) t.Errorf("Complete(%q) = %v; want %v", tt.in, g, tt.complete)
} }
if g := ref.CompleteWithoutBuild(); g != tt.completeWithoutBuild { if g := p.CompleteWithoutBuild(); g != tt.completeWithoutBuild {
t.Errorf("CompleteWithoutBuild(%q) = %v; want %v", tt.in, g, tt.completeWithoutBuild) t.Errorf("CompleteWithoutBuild(%q) = %v; want %v", tt.in, g, tt.completeWithoutBuild)
} }
}) })
} }
} }
func TestRefStringVariants(t *testing.T) { func TestPathStringVariants(t *testing.T) {
cases := []struct { cases := []struct {
in string in string
nameAndTag string nameAndTag string
@ -116,19 +116,19 @@ func TestRefStringVariants(t *testing.T) {
for _, tt := range cases { for _, tt := range cases {
t.Run(tt.in, func(t *testing.T) { t.Run(tt.in, func(t *testing.T) {
ref := ParseRef(tt.in) p := ParsePath(tt.in)
t.Logf("ParseRef(%q) = %#v", tt.in, ref) t.Logf("ParsePath(%q) = %#v", tt.in, p)
if g := ref.NameAndTag(); g != tt.nameAndTag { if g := p.NameAndTag(); g != tt.nameAndTag {
t.Errorf("NameAndTag(%q) = %q; want %q", tt.in, g, tt.nameAndTag) t.Errorf("NameAndTag(%q) = %q; want %q", tt.in, g, tt.nameAndTag)
} }
if g := ref.NameTagAndBuild(); g != tt.nameTagAndBuild { if g := p.NameTagAndBuild(); g != tt.nameTagAndBuild {
t.Errorf("NameTagAndBuild(%q) = %q; want %q", tt.in, g, tt.nameTagAndBuild) t.Errorf("NameTagAndBuild(%q) = %q; want %q", tt.in, g, tt.nameTagAndBuild)
} }
}) })
} }
} }
func TestRefFull(t *testing.T) { func TestPathFull(t *testing.T) {
const empty = "!(MISSING DOMAIN)/!(MISSING NAMESPACE)/!(MISSING NAME):!(MISSING TAG)+!(MISSING BUILD)" const empty = "!(MISSING DOMAIN)/!(MISSING NAMESPACE)/!(MISSING NAME):!(MISSING TAG)+!(MISSING BUILD)"
cases := []struct { cases := []struct {
@ -151,53 +151,53 @@ func TestRefFull(t *testing.T) {
for _, tt := range cases { for _, tt := range cases {
t.Run(tt.in, func(t *testing.T) { t.Run(tt.in, func(t *testing.T) {
ref := ParseRef(tt.in) p := ParsePath(tt.in)
t.Logf("ParseRef(%q) = %#v", tt.in, ref) t.Logf("ParsePath(%q) = %#v", tt.in, p)
if g := ref.Full(); g != tt.wantFull { if g := p.Full(); g != tt.wantFull {
t.Errorf("Full(%q) = %q; want %q", tt.in, g, tt.wantFull) t.Errorf("Full(%q) = %q; want %q", tt.in, g, tt.wantFull)
} }
}) })
} }
} }
func TestParseRefAllocs(t *testing.T) { func TestParsePathAllocs(t *testing.T) {
// test allocations // test allocations
var r Ref var r Path
allocs := testing.AllocsPerRun(1000, func() { allocs := testing.AllocsPerRun(1000, func() {
r = ParseRef("example.com/mistral:7b+Q4_0") r = ParsePath("example.com/mistral:7b+Q4_0")
}) })
_ = r _ = r
if allocs > 0 { if allocs > 0 {
t.Errorf("ParseRef allocs = %v; want 0", allocs) t.Errorf("ParsePath allocs = %v; want 0", allocs)
} }
} }
func BenchmarkParseRef(b *testing.B) { func BenchmarkParsePath(b *testing.B) {
b.ReportAllocs() b.ReportAllocs()
var r Ref var r Path
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
r = ParseRef("example.com/mistral:7b+Q4_0") r = ParsePath("example.com/mistral:7b+Q4_0")
} }
_ = r _ = r
} }
func FuzzParseRef(f *testing.F) { func FuzzParsePath(f *testing.F) {
f.Add("example.com/mistral:7b+Q4_0") f.Add("example.com/mistral:7b+Q4_0")
f.Add("example.com/mistral:7b+q4_0") f.Add("example.com/mistral:7b+q4_0")
f.Add("example.com/mistral:7b+x") f.Add("example.com/mistral:7b+x")
f.Add("x/y/z:8n+I") f.Add("x/y/z:8n+I")
f.Fuzz(func(t *testing.T, s string) { f.Fuzz(func(t *testing.T, s string) {
r0 := ParseRef(s) r0 := ParsePath(s)
if !r0.Valid() { if !r0.Valid() {
if r0 != (Ref{}) { if r0 != (Path{}) {
t.Errorf("expected invalid ref to be zero value; got %#v", r0) t.Errorf("expected invalid path to be zero value; got %#v", r0)
} }
t.Skipf("invalid ref: %q", s) t.Skipf("invalid path: %q", s)
} }
for _, p := range r0.Parts() { for _, p := range r0.Parts() {
if len(p) > MaxRefLength { if len(p) > MaxPathLength {
t.Errorf("part too long: %q", p) t.Errorf("part too long: %q", p)
} }
} }
@ -206,7 +206,7 @@ func FuzzParseRef(f *testing.F) {
t.Errorf("String() did not round-trip with case insensitivity: %q\ngot = %q\nwant = %q", s, r0.String(), s) t.Errorf("String() did not round-trip with case insensitivity: %q\ngot = %q\nwant = %q", s, r0.String(), s)
} }
r1 := ParseRef(r0.String()) r1 := ParsePath(r0.String())
if r0 != r1 { if r0 != r1 {
t.Errorf("round-trip mismatch: %+v != %+v", r0, r1) t.Errorf("round-trip mismatch: %+v != %+v", r0, r1)
} }
@ -216,8 +216,8 @@ func FuzzParseRef(f *testing.F) {
func ExampleMerge() { func ExampleMerge() {
r := Merge( r := Merge(
ParseRef("mistral"), ParsePath("mistral"),
ParseRef("registry.ollama.com/XXXXX:latest+Q4_0"), ParsePath("registry.ollama.com/XXXXX:latest+Q4_0"),
) )
fmt.Println(r) fmt.Println(r)

View File

@ -18,7 +18,7 @@ type Layer struct {
} }
type PushRequest struct { type PushRequest struct {
Ref string `json:"ref"` Name string `json:"ref"`
Manifest json.RawMessage `json:"manifest"` Manifest json.RawMessage `json:"manifest"`
// Parts is a list of upload parts that the client upload in the previous // Parts is a list of upload parts that the client upload in the previous

View File

@ -32,7 +32,7 @@ func (c *Client) Push(ctx context.Context, ref string, manifest []byte, p *PushP
p = cmp.Or(p, &PushParams{}) p = cmp.Or(p, &PushParams{})
// TODO(bmizerany): backoff // TODO(bmizerany): backoff
v, err := ollama.Do[apitype.PushResponse](ctx, c.oclient(), "POST", "/v1/push", &apitype.PushRequest{ v, err := ollama.Do[apitype.PushResponse](ctx, c.oclient(), "POST", "/v1/push", &apitype.PushRequest{
Ref: ref, Name: ref,
Manifest: manifest, Manifest: manifest,
CompleteParts: p.CompleteParts, CompleteParts: p.CompleteParts,
}) })

View File

@ -14,8 +14,8 @@ import (
"strconv" "strconv"
"time" "time"
"bllamo.com/build/blob"
"bllamo.com/client/ollama" "bllamo.com/client/ollama"
"bllamo.com/model"
"bllamo.com/oweb" "bllamo.com/oweb"
"bllamo.com/registry/apitype" "bllamo.com/registry/apitype"
"bllamo.com/utils/upload" "bllamo.com/utils/upload"
@ -82,9 +82,9 @@ func (s *Server) handlePush(w http.ResponseWriter, r *http.Request) error {
return err return err
} }
ref := blob.ParseRef(pr.Ref) mp := model.ParsePath(pr.Name)
if !ref.Complete() { if !mp.Complete() {
return oweb.Invalid("name", pr.Ref, "must be complete") return oweb.Invalid("name", pr.Name, "must be complete")
} }
m, err := oweb.DecodeUserJSON[apitype.Manifest]("manifest", bytes.NewReader(pr.Manifest)) m, err := oweb.DecodeUserJSON[apitype.Manifest]("manifest", bytes.NewReader(pr.Manifest))
@ -205,7 +205,7 @@ func (s *Server) handlePush(w http.ResponseWriter, r *http.Request) error {
if len(requirements) == 0 { if len(requirements) == 0 {
// Commit the manifest // Commit the manifest
body := bytes.NewReader(pr.Manifest) body := bytes.NewReader(pr.Manifest)
path := path.Join("manifests", path.Join(ref.Parts()...)) path := path.Join("manifests", path.Join(mp.Parts()...))
_, err := s.mc().PutObject(r.Context(), bucketTODO, path, body, int64(len(pr.Manifest)), minio.PutObjectOptions{}) _, err := s.mc().PutObject(r.Context(), bucketTODO, path, body, int64(len(pr.Manifest)), minio.PutObjectOptions{})
if err != nil { if err != nil {
return err return err