add model_format config var

Go is required and not installed by default.

.dockerignore

@@ -1,8 +1,4 @@
-build
-llama/build
-.venv
 .vscode
 ollama
 app
-web
-.env
+llm/llama.cpp/ggml

.gitignore

@@ -5,4 +5,4 @@
 .swp
 dist
 ollama
-/ggml-metal.metal
+ggml-metal.metal

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "llm/llama.cpp/ggml"]
+	path = llm/llama.cpp/ggml
+	url = https://github.com/ggerganov/llama.cpp.git
+	ignore = dirty

Dockerfile

@@ -1,15 +1,21 @@
-FROM golang:1.20
+FROM golang:alpine
+
 WORKDIR /go/src/github.com/jmorganca/ollama
+RUN apk add --no-cache git build-base cmake
+
 COPY . .
-RUN CGO_ENABLED=1 go build -ldflags '-linkmode external -extldflags "-static"' .
+RUN go generate ./... && go build -ldflags '-linkmode external -extldflags "-static"' .
 
 FROM alpine
-COPY --from=0 /go/src/github.com/jmorganca/ollama/ollama /bin/ollama
-EXPOSE 11434
+ENV OLLAMA_HOST 0.0.0.0
+RUN apk add --no-cache libstdc++
+
 ARG USER=ollama
 ARG GROUP=ollama
-RUN addgroup -g 1000 $GROUP && adduser -u 1000 -DG $GROUP $USER
+RUN addgroup $GROUP && adduser -D -G $GROUP $USER
+
+COPY --from=0 /go/src/github.com/jmorganca/ollama/ollama /bin/ollama
+
 USER $USER:$GROUP
 ENTRYPOINT ["/bin/ollama"]
-ENV OLLAMA_HOST 0.0.0.0
 CMD ["serve"]

README.md

@@ -50,7 +50,15 @@ Here are some example open-source models that can be downloaded:
 
 ## Examples
 
-### Run a model
+### Pull a public model
+
+```
+ollama pull llama2
+```
+
+> This command can also be used to update a local model. Only updated changes will be pulled.
+
+### Run a model interactively
 
 ```
 ollama run llama2
@@ -67,7 +75,39 @@ For multiline input, you can wrap text with `"""`:
 I'm a basic program that prints the famous "Hello, world!" message to the console.
 ```
 
-### Create a custom model
+### Run a model non-interactively
+
+```
+$ ollama run llama2 'tell me a joke'
+ Sure! Here's a quick one:
+ Why did the scarecrow win an award? Because he was outstanding in his field!
+```
+
+```
+$ cat <<EOF >prompts.txt
+tell me a joke about llamas
+tell me another one
+EOF
+$ ollama run llama2 <prompts.txt
+>>> tell me a joke about llamas
+ Why did the llama refuse to play hide-and-seek?
+ nobody likes to be hided!
+
+>>> tell me another one
+ Sure, here's another one:
+
+Why did the llama go to the bar?
+To have a hay-often good time!
+```
+
+### Run a model on contents of a text file
+
+```
+$ ollama run llama2 "summarize this file:" "$(cat README.md)"
+ Ollama is a lightweight, extensible framework for building and running language models on the local machine. It provides a simple API for creating, running, and managing models, as well as a library of pre-built models that can be easily used in a variety of applications.
+```
+
+### Customize a model
 
 Pull a base model:
 
@@ -75,8 +115,6 @@ Pull a base model:
 ollama pull llama2
 ```
 
-> To update a model to the latest version, run `ollama pull llama2` again. The model will be updated (if necessary).
-
 Create a `Modelfile`:
 
 ```
@@ -102,23 +140,23 @@ Hello! It's your friend Mario.
 
 For more examples, see the [examples](./examples) directory. For more information on creating a Modelfile, see the [Modelfile](./docs/modelfile.md) documentation.
 
-### Pull a model from the registry
-
-```
-ollama pull orca-mini
-```
-
 ### Listing local models
 
 ```
 ollama list
 ```
 
+### Removing local models
+
+```
+ollama rm llama2
+```
+
 ## Model packages
 
 ### Overview
 
-Ollama bundles model weights, configuration, and data into a single package, defined by a [Modelfile](./docs/modelfile.md).
+Ollama bundles model weights, configurations, and data into a single package, defined by a [Modelfile](./docs/modelfile.md).
 
 <picture>
   <source media="(prefers-color-scheme: dark)" height="480" srcset="https://github.com/jmorganca/ollama/assets/251292/2fd96b5f-191b-45c1-9668-941cfad4eb70">
@@ -127,19 +165,27 @@ Ollama bundles model weights, configuration, and data into a single package, def
 
 ## Building
 
-You will also need a C/C++ compiler such as GCC for MacOS and Linux or Mingw-w64 GCC for Windows.
+Install `cmake` and `go`:
 
 ```
+brew install cmake
+brew install go
+```
+
+Then generate dependencies and build:
+
+```
+go generate ./...
 go build .
 ```
 
-To run it start the server:
+Next, start the server:
 
 ```
-./ollama serve &
+./ollama serve
 ```
 
-Finally, run a model!
+Finally, in a separate shell, run a model:
 
 ```
 ./ollama run llama2
@@ -158,7 +204,7 @@ curl -X POST http://localhost:11434/api/generate -d '{
 }'
 ```
 
-## Tools using Ollama
+## Community Projects using Ollama
 
 - [LangChain](https://python.langchain.com/docs/integrations/llms/ollama) and [LangChain.js](https://js.langchain.com/docs/modules/model_io/models/llms/integrations/ollama) with a question-answering [example](https://js.langchain.com/docs/use_cases/question_answering/local_retrieval_qa).
 - [Continue](https://github.com/continuedev/continue) - embeds Ollama inside Visual Studio Code. The extension lets you highlight code to add to the prompt, ask questions in the sidebar, and generate code inline.

api/client.go

@@ -255,6 +255,14 @@ func (c *Client) Delete(ctx context.Context, req *DeleteRequest) error {
 	return nil
 }
 
+func (c *Client) Show(ctx context.Context, req *ShowRequest) (*ShowResponse, error) {
+	var resp ShowResponse
+	if err := c.do(ctx, http.MethodPost, "/api/show", req, &resp); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
 func (c *Client) Heartbeat(ctx context.Context) error {
 	if err := c.do(ctx, http.MethodHead, "/", nil, nil); err != nil {
 		return err

api/types.go

@@ -7,7 +7,6 @@ import (
 	"math"
 	"os"
 	"reflect"
-	"runtime"
 	"strings"
 	"time"
 )
@@ -62,6 +61,18 @@ type DeleteRequest struct {
 	Name string `json:"name"`
 }
 
+type ShowRequest struct {
+	Name string `json:"name"`
+}
+
+type ShowResponse struct {
+	License    string `json:"license,omitempty"`
+	Modelfile  string `json:"modelfile,omitempty"`
+	Parameters string `json:"parameters,omitempty"`
+	Template   string `json:"template,omitempty"`
+	System     string `json:"system,omitempty"`
+}
+
 type CopyRequest struct {
 	Source      string `json:"source"`
 	Destination string `json:"destination"`
@@ -89,10 +100,10 @@ type PushRequest struct {
 }
 
 type ListResponse struct {
-	Models []ListResponseModel `json:"models"`
+	Models []ModelResponse `json:"models"`
 }
 
-type ListResponseModel struct {
+type ModelResponse struct {
 	Name       string    `json:"name"`
 	ModifiedAt time.Time `json:"modified_at"`
 	Size       int       `json:"size"`
@@ -113,8 +124,6 @@ type GenerateResponse struct {
 
 	TotalDuration      time.Duration `json:"total_duration,omitempty"`
 	LoadDuration       time.Duration `json:"load_duration,omitempty"`
-	SampleCount        int           `json:"sample_count,omitempty"`
-	SampleDuration     time.Duration `json:"sample_duration,omitempty"`
 	PromptEvalCount    int           `json:"prompt_eval_count,omitempty"`
 	PromptEvalDuration time.Duration `json:"prompt_eval_duration,omitempty"`
 	EvalCount          int           `json:"eval_count,omitempty"`
@@ -130,15 +139,6 @@ func (r *GenerateResponse) Summary() {
 		fmt.Fprintf(os.Stderr, "load duration:        %v\n", r.LoadDuration)
 	}
 
-	if r.SampleCount > 0 {
-		fmt.Fprintf(os.Stderr, "sample count:         %d token(s)\n", r.SampleCount)
-	}
-
-	if r.SampleDuration > 0 {
-		fmt.Fprintf(os.Stderr, "sample duration:      %s\n", r.SampleDuration)
-		fmt.Fprintf(os.Stderr, "sample rate:          %.2f tokens/s\n", float64(r.SampleCount)/r.SampleDuration.Seconds())
-	}
-
 	if r.PromptEvalCount > 0 {
 		fmt.Fprintf(os.Stderr, "prompt eval count:    %d token(s)\n", r.PromptEvalCount)
 	}
@@ -182,15 +182,16 @@ type Options struct {
 	RopeFrequencyScale float32 `json:"rope_frequency_scale,omitempty"`
 
 	// Predict options
-	RepeatLastN      int      `json:"repeat_last_n,omitempty"`
-	RepeatPenalty    float32  `json:"repeat_penalty,omitempty"`
-	FrequencyPenalty float32  `json:"frequency_penalty,omitempty"`
-	PresencePenalty  float32  `json:"presence_penalty,omitempty"`
-	Temperature      float32  `json:"temperature,omitempty"`
+	NumPredict       int      `json:"num_predict,omitempty"`
 	TopK             int      `json:"top_k,omitempty"`
 	TopP             float32  `json:"top_p,omitempty"`
 	TFSZ             float32  `json:"tfs_z,omitempty"`
 	TypicalP         float32  `json:"typical_p,omitempty"`
+	RepeatLastN      int      `json:"repeat_last_n,omitempty"`
+	Temperature      float32  `json:"temperature,omitempty"`
+	RepeatPenalty    float32  `json:"repeat_penalty,omitempty"`
+	PresencePenalty  float32  `json:"presence_penalty,omitempty"`
+	FrequencyPenalty float32  `json:"frequency_penalty,omitempty"`
 	Mirostat         int      `json:"mirostat,omitempty"`
 	MirostatTau      float32  `json:"mirostat_tau,omitempty"`
 	MirostatEta      float32  `json:"mirostat_eta,omitempty"`
@@ -314,7 +315,7 @@ func DefaultOptions() Options {
 		MirostatEta:      0.1,
 		PenalizeNewline:  true,
 
-		NumThread: runtime.NumCPU(),
+		NumThread: 0, // let the runtime decide
 	}
 }
 

app/forge.config.ts

@@ -27,7 +27,6 @@ const config: ForgeConfig = {
       path.join(__dirname, './assets/iconDarkTemplate@2x.png'),
       path.join(__dirname, './assets/iconDarkUpdateTemplate.png'),
       path.join(__dirname, './assets/iconDarkUpdateTemplate@2x.png'),
-      ...(process.platform === 'darwin' ? ['../llm/ggml-metal.metal'] : []),
     ],
     ...(process.env.SIGN
       ? {

app/src/index.ts

@@ -158,7 +158,7 @@ function restart() {
 app.on('before-quit', () => {
   if (proc) {
     proc.off('exit', restart)
-    proc.kill()
+    proc.kill('SIGINT') // send SIGINT signal to the server, which also stops any loaded llms
   }
 })
 

cmd/cmd.go

@@ -230,6 +230,84 @@ func DeleteHandler(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
+func ShowHandler(cmd *cobra.Command, args []string) error {
+	client, err := api.FromEnv()
+	if err != nil {
+		return err
+	}
+
+	if len(args) != 1 {
+		return errors.New("missing model name")
+	}
+
+	license, errLicense := cmd.Flags().GetBool("license")
+	modelfile, errModelfile := cmd.Flags().GetBool("modelfile")
+	parameters, errParams := cmd.Flags().GetBool("parameters")
+	system, errSystem := cmd.Flags().GetBool("system")
+	template, errTemplate := cmd.Flags().GetBool("template")
+
+	for _, boolErr := range []error{errLicense, errModelfile, errParams, errSystem, errTemplate} {
+		if boolErr != nil {
+			return errors.New("error retrieving flags")
+		}
+	}
+
+	flagsSet := 0
+	showType := ""
+
+	if license {
+		flagsSet++
+		showType = "license"
+	}
+
+	if modelfile {
+		flagsSet++
+		showType = "modelfile"
+	}
+
+	if parameters {
+		flagsSet++
+		showType = "parameters"
+	}
+
+	if system {
+		flagsSet++
+		showType = "system"
+	}
+
+	if template {
+		flagsSet++
+		showType = "template"
+	}
+
+	if flagsSet > 1 {
+		return errors.New("only one of '--license', '--modelfile', '--parameters', '--system', or '--template' can be specified")
+	} else if flagsSet == 0 {
+		return errors.New("one of '--license', '--modelfile', '--parameters', '--system', or '--template' must be specified")
+	}
+
+	req := api.ShowRequest{Name: args[0]}
+	resp, err := client.Show(context.Background(), &req)
+	if err != nil {
+		return err
+	}
+
+	switch showType {
+	case "license":
+		fmt.Println(resp.License)
+	case "modelfile":
+		fmt.Println(resp.Modelfile)
+	case "parameters":
+		fmt.Println(resp.Parameters)
+	case "system":
+		fmt.Println(resp.System)
+	case "template":
+		fmt.Println(resp.Template)
+	}
+
+	return nil
+}
+
 func CopyHandler(cmd *cobra.Command, args []string) error {
 	client, err := api.FromEnv()
 	if err != nil {
@@ -377,20 +455,6 @@ func generate(cmd *cobra.Command, model, prompt string) error {
 	return nil
 }
 
-func showLayer(l *server.Layer) {
-	filename, err := server.GetBlobsPath(l.Digest)
-	if err != nil {
-		fmt.Println("Couldn't get layer's path")
-		return
-	}
-	bts, err := os.ReadFile(filename)
-	if err != nil {
-		fmt.Println("Couldn't read layer")
-		return
-	}
-	fmt.Println(string(bts))
-}
-
 func generateInteractive(cmd *cobra.Command, model string) error {
 	home, err := os.UserHomeDir()
 	if err != nil {
@@ -413,6 +477,8 @@ func generateInteractive(cmd *cobra.Command, model string) error {
 		),
 		readline.PcItem("/show",
 			readline.PcItem("license"),
+			readline.PcItem("modelfile"),
+			readline.PcItem("parameters"),
 			readline.PcItem("system"),
 			readline.PcItem("template"),
 		),
@@ -522,42 +588,28 @@ func generateInteractive(cmd *cobra.Command, model string) error {
 		case strings.HasPrefix(line, "/show"):
 			args := strings.Fields(line)
 			if len(args) > 1 {
-				mp := server.ParseModelPath(model)
+				resp, err := server.GetModelInfo(model)
 				if err != nil {
-					return err
+					fmt.Println("error: couldn't get model")
+					continue
 				}
 
-				manifest, _, err := server.GetManifest(mp)
-				if err != nil {
-					fmt.Println("error: couldn't get a manifest for this model")
-					continue
-				}
 				switch args[1] {
 				case "license":
-					for _, l := range manifest.Layers {
-						if l.MediaType == "application/vnd.ollama.image.license" {
-							showLayer(l)
-						}
-					}
-					continue
+					fmt.Println(resp.License)
+				case "modelfile":
+					fmt.Println(resp.Modelfile)
+				case "parameters":
+					fmt.Println(resp.Parameters)
 				case "system":
-					for _, l := range manifest.Layers {
-						if l.MediaType == "application/vnd.ollama.image.system" {
-							showLayer(l)
-						}
-					}
-					continue
+					fmt.Println(resp.System)
 				case "template":
-					for _, l := range manifest.Layers {
-						if l.MediaType == "application/vnd.ollama.image.template" {
-							showLayer(l)
-						}
-					}
-					continue
+					fmt.Println(resp.Template)
 				default:
-					usage()
-					continue
+					fmt.Println("error: unknown command")
 				}
+
+				continue
 			} else {
 				usage()
 				continue
@@ -749,6 +801,20 @@ func NewCLI() *cobra.Command {
 
 	createCmd.Flags().StringP("file", "f", "Modelfile", "Name of the Modelfile (default \"Modelfile\")")
 
+	showCmd := &cobra.Command{
+		Use:     "show MODEL",
+		Short:   "Show information for a model",
+		Args:    cobra.MinimumNArgs(1),
+		PreRunE: checkServerHeartbeat,
+		RunE:    ShowHandler,
+	}
+
+	showCmd.Flags().Bool("license", false, "Show license of a model")
+	showCmd.Flags().Bool("modelfile", false, "Show Modelfile of a model")
+	showCmd.Flags().Bool("parameters", false, "Show parameters of a model")
+	showCmd.Flags().Bool("template", false, "Show template of a model")
+	showCmd.Flags().Bool("system", false, "Show system prompt of a model")
+
 	runCmd := &cobra.Command{
 		Use:     "run MODEL [PROMPT]",
 		Short:   "Run a model",
@@ -814,6 +880,7 @@ func NewCLI() *cobra.Command {
 	rootCmd.AddCommand(
 		serveCmd,
 		createCmd,
+		showCmd,
 		runCmd,
 		pullCmd,
 		pushCmd,

docs/api.md

@@ -238,6 +238,10 @@ Generate embeddings from a model
 - `model`: name of model to generate embeddings from
 - `prompt`: text to generate embeddings for
 
+Advanced parameters:
+
+- `options`: additional model parameters listed in the documentation for the [Modelfile](./modelfile.md#valid-parameters-and-values) such as `temperature`
+
 ### Request
 
 ```

docs/development.md

@@ -1,19 +1,21 @@
 # Development
 
+- Install cmake or (optionally, required tools for GPUs)
+- run `go generate ./...`
+- run `go build .`
+
 Install required tools:
 
 ```
-brew install go
+brew install go cmake gcc
 ```
 
-Enable CGO:
+Get the required libraries:
 
 ```
-export CGO_ENABLED=1
+go generate ./...
 ```
 
-You will also need a C/C++ compiler such as GCC for MacOS and Linux or Mingw-w64 GCC for Windows.
-
 Then build ollama:
 
 ```

examples/10tweets/Modelfile

@@ -0,0 +1,7 @@
+# Modelfile for creating a list of ten tweets from a topic
+# Run `ollama create 10tweets -f ./Modelfile` and then `ollama run 10tweets` and enter a topic
+
+FROM llama2
+SYSTEM """
+You are a content marketer who needs to come up with 10 short but succinct tweets. The answer should be a list of ten tweets. Each tweet can have a maximum of 280 characters and should include hashtags. Each user input will be a subject and you should expand it in ten creative ways. Never stop after just one tweet. Always include ten. 
+"""

examples/dockerit/Modelfile

@@ -1,6 +1,6 @@
 FROM llama2
 SYSTEM """
-You are an experience Devops engineer focused on docker. When given specifications for a particular need or application you know the best way to host that within a docker container. For instance if someone tells you they want an nginx server to host files located at /web you will answer as follows
+You are an experienced Devops engineer focused on docker. When given specifications for a particular need or application you know the best way to host that within a docker container. For instance if someone tells you they want an nginx server to host files located at /web you will answer as follows
 
 ---start
 FROM nginx:alpine
@@ -16,5 +16,5 @@ ENV POSTGRES_PASSWORD=abc123
 EXPOSE 5432
 ---end
 
-Again it's just the contents of the dockerfile an nothing else.
-"""
+Again it's just the contents of the dockerfile and nothing else.
+"""

examples/sentiments/Modelfile

@@ -0,0 +1,28 @@
+# Modelfile for creating a sentiment analyzer. 
+# Run `ollama create sentiments -f pathtofile` and then `ollama run sentiments` and enter a topic
+
+FROM orca
+TEMPLATE """
+{{- if .First }}
+### System:
+{{ .System }}
+{{- end }}
+### User: 
+I hate it when my phone dies
+### Response: 
+NEGATIVE
+### User: 
+He is awesome
+### Response: 
+POSITIVE
+### User: 
+This is the link to the article
+### Response: 
+NEUTRAL
+### User:
+{{ .Prompt }}
+
+### Response:
+"""
+
+SYSTEM """You are a sentiment analyzer. You will receive text and output only one word, either POSITIVE or NEGATIVE or NEUTRAL, depending on the sentiment of the text."""

examples/sentiments/Readme.md

@@ -0,0 +1,25 @@
+# Sentiments Modelfile
+
+This is a simple sentiments analyzer using the Orca model. When you pull Orca from the registry, it has a Template already defined that looks like this:
+
+```Modelfile
+{{- if .First }}
+### System:
+{{ .System }}
+{{- end }}
+
+### User:
+{{ .Prompt }}
+
+### Response:
+```
+
+If we just wanted to have the text:
+
+```Plaintext
+You are a sentiment analyzer. You will receive text and output only one word, either POSITIVE or NEGATIVE or NEUTRAL, depending on the sentiment of the text.
+```
+
+then we could have put this in a SYSTEM block. But we want to provide examples which require updating the full Template. Any Modelfile you create will inherit all the settings from the source model. But in this example, we are overriding the Template.
+
+When providing examples for the input and output, you should include the way the model usually provides information. Since the Orca model expects a user prompt to appear after ### User: and the response is after ### Response, we should format our examples like that as well. If we were using the Llama 2 model, the format would be a bit different.

format/openssh.go

@@ -10,15 +10,11 @@ package format
 
 import (
 	"crypto"
-	"crypto/ecdsa"
 	"crypto/ed25519"
-	"crypto/elliptic"
 	"crypto/rand"
-	"crypto/rsa"
 	"encoding/binary"
 	"encoding/pem"
 	"fmt"
-	"math/big"
 
 	"golang.org/x/crypto/ssh"
 )
@@ -41,25 +37,6 @@ type openSSHPrivateKey struct {
 	Rest    []byte `ssh:"rest"`
 }
 
-type openSSHRSAPrivateKey struct {
-	N       *big.Int
-	E       *big.Int
-	D       *big.Int
-	Iqmp    *big.Int
-	P       *big.Int
-	Q       *big.Int
-	Comment string
-	Pad     []byte `ssh:"rest"`
-}
-
-type openSSHECDSAPrivateKey struct {
-	Curve   string
-	Pub     []byte
-	D       *big.Int
-	Comment string
-	Pad     []byte `ssh:"rest"`
-}
-
 type openSSHEd25519PrivateKey struct {
 	Pub     []byte
 	Priv    []byte
@@ -85,64 +62,6 @@ func OpenSSHPrivateKey(key crypto.PrivateKey, comment string) (*pem.Block, error
 	}
 
 	switch k := key.(type) {
-	case *rsa.PrivateKey:
-		e := new(big.Int).SetInt64(int64(k.E))
-
-		key := openSSHRSAPrivateKey{
-			N:       k.N,
-			E:       e,
-			D:       k.D,
-			Iqmp:    k.Precomputed.Qinv,
-			P:       k.Primes[0],
-			Q:       k.Primes[1],
-			Comment: comment,
-		}
-
-		pk1.Keytype = ssh.KeyAlgoRSA
-		pk1.Rest = ssh.Marshal(key)
-
-		w.PubKey = ssh.Marshal(struct {
-			KeyType string
-			E       *big.Int
-			N       *big.Int
-		}{
-			ssh.KeyAlgoRSA, e, k.N,
-		})
-	case *ecdsa.PrivateKey:
-		var curve, keytype string
-		switch name := k.Curve.Params().Name; name {
-		case "P-256":
-			curve = "nistp256"
-			keytype = ssh.KeyAlgoECDSA256
-		case "P-384":
-			curve = "nistp384"
-			keytype = ssh.KeyAlgoECDSA384
-		case "P-521":
-			curve = "nistp521"
-			keytype = ssh.KeyAlgoECDSA521
-		default:
-			return nil, fmt.Errorf("ssh: unknown curve %q", name)
-		}
-
-		pub := elliptic.Marshal(k.Curve, k.X, k.Y)
-
-		key := openSSHECDSAPrivateKey{
-			Curve:   curve,
-			Pub:     pub,
-			D:       k.D,
-			Comment: comment,
-		}
-
-		pk1.Keytype = keytype
-		pk1.Rest = ssh.Marshal(key)
-
-		w.PubKey = ssh.Marshal(struct {
-			KeyType string
-			Curve   string
-			Pub     []byte
-		}{
-			keytype, curve, pub,
-		})
 	case ed25519.PrivateKey:
 		pub, priv := k[32:], k
 		key := openSSHEd25519PrivateKey{

go.mod

@@ -38,9 +38,10 @@ require (
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.11 // indirect
 	golang.org/x/arch v0.3.0 // indirect
-	golang.org/x/crypto v0.10.0 // indirect
+	golang.org/x/crypto v0.10.0
+	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63
 	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/sys v0.10.0 // indirect
+	golang.org/x/sys v0.11.0 // indirect
 	golang.org/x/term v0.10.0
 	golang.org/x/text v0.10.0 // indirect
 	gonum.org/v1/gonum v0.13.0

go.sum

@@ -120,6 +120,9 @@ golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM=
 golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
+golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug=
+golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ=
+golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
@@ -130,8 +133,8 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
-golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
 golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=

llm/ggml-alloc.c

@@ -1,575 +0,0 @@
-/**
- * llama.cpp - git 3ebb00935f3f0522b75df49c2769ab1774b91380
- *
- * MIT License
- *
- * Copyright (c) 2023 Georgi Gerganov
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include "ggml-alloc.h"
-#include "ggml.h"
-#include <assert.h>
-#include <stdarg.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#define UNUSED(x) (void)(x)
-#define MAX(a, b) ((a) > (b) ? (a) : (b))
-
-//#define GGML_ALLOCATOR_DEBUG
-
-//#define AT_PRINTF printf
-#define AT_PRINTF(...) ((void)0)
-
-struct hash_node {
-    struct ggml_tensor * t;
-    int n_children;
-    int n_views;
-};
-
-static size_t hash(void * p) {
-    return (size_t)p % GGML_GRAPH_HASHTABLE_SIZE;
-}
-
-static struct hash_node * hash_get(struct hash_node hash_table[], struct ggml_tensor * t) {
-    size_t h = hash(t);
-
-    // linear probing
-    size_t i = h;
-    while (hash_table[i].t != NULL) {
-        if (hash_table[i].t == t) {
-            return &hash_table[i];
-        }
-        i = (i + 1) % GGML_GRAPH_HASHTABLE_SIZE;
-        if (i == h) {
-            // hash table is full
-            GGML_ASSERT(false);
-        }
-    }
-
-    hash_table[i].t = t;
-    return &hash_table[i];
-}
-
-// TODO: GGML_PAD ?
-static size_t aligned_offset(const void * buffer, size_t offset, size_t alignment) {
-    assert(alignment && !(alignment & (alignment - 1))); // power of 2
-    size_t align = (alignment - (((uintptr_t)buffer + offset) % alignment)) % alignment;
-    return offset + align;
-}
-
-struct free_block {
-    void * addr;
-    size_t size;
-};
-
-#define MAX_FREE_BLOCKS 128
-
-struct ggml_allocr {
-    void * data;
-    size_t size;
-    size_t alignment;
-    int n_free_blocks;
-    struct free_block free_blocks[MAX_FREE_BLOCKS];
-    struct hash_node hash_table[GGML_GRAPH_HASHTABLE_SIZE];
-    size_t max_size;
-    bool measure;
-
-#ifdef GGML_ALLOCATOR_DEBUG
-    struct ggml_tensor * allocated_tensors[1024];
-#endif
-};
-
-#ifdef GGML_ALLOCATOR_DEBUG
-static void add_allocated_tensor(struct ggml_allocator * alloc, struct ggml_tensor * tensor) {
-    for (int i = 0; i < 1024; i++) {
-        if (alloc->allocated_tensors[i] == NULL) {
-            alloc->allocated_tensors[i] = tensor;
-            return;
-        }
-    }
-    GGML_ASSERT(!"out of allocated_tensors");
-}
-static void remove_allocated_tensor(struct ggml_allocator * alloc, struct ggml_tensor * tensor) {
-    for (int i = 0; i < 1024; i++) {
-        if (alloc->allocated_tensors[i] == tensor ||
-            (alloc->allocated_tensors[i] != NULL && alloc->allocated_tensors[i]->data == tensor->data)) {
-            alloc->allocated_tensors[i] = NULL;
-            return;
-        }
-    }
-    printf("tried to free tensor %s not found\n", tensor->name);
-    GGML_ASSERT(!"tensor not found");
-}
-#endif
-
-
-static size_t ggml_allocator_get_alloc_size(struct ggml_allocr * alloc, struct ggml_tensor * tensor) {
-    return ggml_nbytes(tensor);
-
-    UNUSED(alloc);
-}
-
-void ggml_allocr_alloc(struct ggml_allocr * alloc, struct ggml_tensor * tensor) {
-    size_t size = ggml_allocator_get_alloc_size(alloc, tensor);
-    size = aligned_offset(NULL, size, alloc->alignment);
-
-    AT_PRINTF("%s: allocating %s (%zu bytes) - ", __func__, tensor->name, size);
-
-    size_t max_avail = 0;
-
-    // find the best fitting free block
-    int best_fit_block = -1;
-    size_t best_fit_size = SIZE_MAX;
-    for (int i = 0; i < alloc->n_free_blocks; i++) {
-        struct free_block * block = &alloc->free_blocks[i];
-        max_avail = MAX(max_avail, block->size);
-        if (block->size >= size && block->size <= best_fit_size) {
-            best_fit_block = i;
-            best_fit_size = block->size;
-        }
-    }
-
-    AT_PRINTF("block %d\n", best_fit_block);
-
-    if (best_fit_block == -1) {
-        fprintf(stderr, "%s: not enough space in the buffer (needed %zu, largest block available %zu)\n",
-                __func__, size, max_avail);
-        GGML_ASSERT(!"not enough space in the buffer");
-        return;
-    }
-    struct free_block * block = &alloc->free_blocks[best_fit_block];
-    void * addr = block->addr;
-    block->addr = (char*)block->addr + size;
-    block->size -= size;
-    if (block->size == 0) {
-        // remove block if empty
-        alloc->n_free_blocks--;
-        for (int j = best_fit_block; j < alloc->n_free_blocks; j++) {
-            alloc->free_blocks[j] = alloc->free_blocks[j+1];
-        }
-    }
-
-    tensor->data = addr;
-
-#ifdef GGML_ALLOCATOR_DEBUG
-    add_allocated_tensor(alloc, tensor);
-    size_t cur_max = (char*)addr - (char*)alloc->data + size;
-    if (cur_max > alloc->max_size) {
-        printf("max_size = %.2f MB: tensors: ", cur_max / 1024.0 / 1024.0);
-        for (int i = 0; i < 1024; i++) {
-            if (alloc->allocated_tensors[i]) {
-                printf("%s (%.2f MB) ", alloc->allocated_tensors[i]->name, ggml_nbytes(alloc->allocated_tensors[i]) / 1024.0 / 1024.0);
-            }
-        }
-        printf("\n");
-    }
-#endif
-
-    alloc->max_size = MAX(alloc->max_size, (char*)addr - (char*)alloc->data + size);
-}
-
-// this is a very naive implementation, but for our case the number of free blocks should be very small
-static void ggml_allocator_free_tensor(struct ggml_allocr * alloc, struct ggml_tensor * tensor) {
-    void * ptr = tensor->data;
-
-    if (ptr < alloc->data || (char*)ptr >= (char*)alloc->data + alloc->max_size) {
-        // the tensor was not allocated in this buffer
-        // this can happen because the graph allocator will try to free weights and other tensors from different buffers
-        // the easiest way to deal with this is just to ignore it
-        return;
-    }
-
-    size_t size = ggml_allocator_get_alloc_size(alloc, tensor);
-    size = aligned_offset(NULL, size, alloc->alignment);
-    AT_PRINTF("%s: freeing %s (%zu bytes) - n_free_blocks = %d\n", __func__, tensor->name, size, alloc->n_free_blocks);
-
-#ifdef GGML_ALLOCATOR_DEBUG
-    remove_allocated_tensor(alloc, tensor);
-#endif
-
-    // see if we can merge with an existing block
-    for (int i = 0; i < alloc->n_free_blocks; i++) {
-        struct free_block * block = &alloc->free_blocks[i];
-        // check if ptr is at the end of the block
-        if ((char*)block->addr + block->size == ptr) {
-            block->size += size;
-            // check if we can merge with the next block
-            if (i < alloc->n_free_blocks - 1 && (char*)block->addr + block->size == alloc->free_blocks[i+1].addr) {
-                block->size += alloc->free_blocks[i+1].size;
-                alloc->n_free_blocks--;
-                for (int j = i+1; j < alloc->n_free_blocks; j++) {
-                    alloc->free_blocks[j] = alloc->free_blocks[j+1];
-                }
-            }
-            return;
-        }
-        // check if ptr is at the beginning of the block
-        if ((char*)ptr + size == block->addr) {
-            block->addr = ptr;
-            block->size += size;
-            // check if we can merge with the previous block
-            if (i > 0 && (char*)alloc->free_blocks[i-1].addr + alloc->free_blocks[i-1].size == block->addr) {
-                alloc->free_blocks[i-1].size += block->size;
-                alloc->n_free_blocks--;
-                for (int j = i; j < alloc->n_free_blocks; j++) {
-                    alloc->free_blocks[j] = alloc->free_blocks[j+1];
-                }
-            }
-            return;
-        }
-    }
-    // otherwise, add a new block
-    GGML_ASSERT(alloc->n_free_blocks < MAX_FREE_BLOCKS && "out of free blocks");
-    // insert the new block in the correct position to keep the array sorted by address (to make merging blocks faster)
-    int insert_pos = 0;
-    while (insert_pos < alloc->n_free_blocks && alloc->free_blocks[insert_pos].addr < ptr) {
-        insert_pos++;
-    }
-    // shift all blocks from insert_pos onward to make room for the new block
-    for (int i = alloc->n_free_blocks; i > insert_pos; i--) {
-        alloc->free_blocks[i] = alloc->free_blocks[i-1];
-    }
-    // insert the new block
-    alloc->free_blocks[insert_pos].addr = ptr;
-    alloc->free_blocks[insert_pos].size = size;
-    alloc->n_free_blocks++;
-}
-
-void ggml_allocr_reset(struct ggml_allocr * alloc) {
-    alloc->n_free_blocks = 1;
-    size_t align_offset = aligned_offset(alloc->data, 0, alloc->alignment);
-    alloc->free_blocks[0].addr = (char *)alloc->data + align_offset;
-    alloc->free_blocks[0].size = alloc->size - align_offset;
-}
-
-struct ggml_allocr * ggml_allocr_new(void * data, size_t size, size_t alignment) {
-    struct ggml_allocr * alloc = (struct ggml_allocr *)malloc(sizeof(struct ggml_allocr) /* + n_free_blocks * sizeof(struct free_block) */);
-
-    *alloc = (struct ggml_allocr){
-        /*.data          = */ data,
-        /*.size          = */ size,
-        /*.alignment     = */ alignment,
-        /*.n_free_blocks = */ 0,
-        /*.free_blocks   = */ {{0}},
-        /*.hash_table    = */ {{0}},
-        /*.max_size      = */ 0,
-        /*.measure       = */ false,
-#ifdef GGML_ALLOCATOR_DEBUG
-        /*.allocated_tensors = */ = {0},
-#endif
-    };
-
-    ggml_allocr_reset(alloc);
-
-    return alloc;
-}
-
-// address and size of the buffer when measuring
-// it needs to be large enough to fit all the tensors, but it cannot overlap with other existing buffers
-static void * const MEASURE_BASE_ADDR = (void *) 0x1000;
-static const size_t MEASURE_MAX_SIZE  = 1ULL<<40; // 1 TB
-
-struct ggml_allocr * ggml_allocr_new_measure(size_t alignment) {
-    struct ggml_allocr * alloc = (struct ggml_allocr *)malloc(sizeof(struct ggml_allocr) /* + n_free_blocks * sizeof(struct free_block) */);
-
-    *alloc = (struct ggml_allocr){
-        /*.data          = */ MEASURE_BASE_ADDR,
-        /*.size          = */ MEASURE_MAX_SIZE,
-        /*.alignment     = */ alignment,
-        /*.n_free_blocks = */ 0,
-        /*.free_blocks   = */ {{0}},
-        /*.hash_table    = */ {{0}},
-        /*.max_size      = */ 0,
-        /*.measure       = */ true,
-#ifdef GGML_ALLOCATOR_DEBUG
-        /*.allocated_tensors = */ = {0},
-#endif
-    };
-
-    ggml_allocr_reset(alloc);
-
-    return alloc;
-}
-
-void ggml_allocr_free(struct ggml_allocr * alloc) {
-    free(alloc);
-}
-
-bool ggml_allocr_is_measure(struct ggml_allocr * alloc) {
-    return alloc->measure;
-}
-
-//////////// compute graph allocator
-
-static bool ggml_is_view(struct ggml_tensor * t) {
-    return t->op == GGML_OP_RESHAPE || t->op == GGML_OP_VIEW || t->op == GGML_OP_TRANSPOSE ||
-           t->op == GGML_OP_PERMUTE || t->op == GGML_OP_CPY;
-}
-
-static bool ggml_are_same_layout(const struct ggml_tensor * a, const struct ggml_tensor * b) {
-    if (a->type != b->type) {
-        return false;
-    }
-    for (int i = 0; i < GGML_MAX_DIMS; i++) {
-        if (a->ne[i] != b->ne[i]) {
-            return false;
-        }
-        if (a->nb[i] != b->nb[i]) {
-            return false;
-        }
-    }
-    return true;
-}
-
-static struct ggml_tensor * get_view_parent(struct ggml_tensor * t) {
-    switch (t->op) {
-        case GGML_OP_PERMUTE:
-        case GGML_OP_RESHAPE:
-        case GGML_OP_TRANSPOSE:
-        case GGML_OP_VIEW:
-            return t->src[0];
-        case GGML_OP_CPY:
-            return t->src[1];
-        default:
-            return NULL;
-    }
-}
-
-static struct ggml_tensor * get_view_source(struct ggml_tensor * t) {
-    struct ggml_tensor * parent = t;
-    do {
-        parent = get_view_parent(parent);
-    } while (ggml_is_view(parent));
-    return parent;
-}
-
-static bool ggml_op_can_inplace(enum ggml_op op) {
-    switch (op) {
-        case GGML_OP_SCALE:
-        case GGML_OP_DIAG_MASK_ZERO:
-        case GGML_OP_DIAG_MASK_INF:
-        case GGML_OP_ADD:
-        case GGML_OP_ADD1:
-        case GGML_OP_ACC:
-        case GGML_OP_SUB:
-        case GGML_OP_MUL:
-        case GGML_OP_DIV:
-        case GGML_OP_SQR:
-        case GGML_OP_SQRT:
-        case GGML_OP_LOG:
-        case GGML_OP_UNARY:
-        case GGML_OP_ROPE:
-        case GGML_OP_RMS_NORM:
-        case GGML_OP_SET:
-        case GGML_OP_SOFT_MAX:
-        case GGML_OP_CONT:
-            return true;
-
-        default:
-            return false;
-    }
-}
-
-static void allocate_node(struct ggml_allocr * alloc, struct ggml_tensor * node) {
-    struct hash_node * ht = alloc->hash_table;
-    if (node->data == NULL) {
-        if (ggml_is_view(node)) {
-            size_t offset;
-            switch(node->op) {
-                case GGML_OP_VIEW:
-                    memcpy(&offset, node->op_params, sizeof(size_t));
-                    node->data = (char *) node->src[0]->data + offset;
-                    break;
-                case GGML_OP_PERMUTE:
-                case GGML_OP_RESHAPE:
-                case GGML_OP_TRANSPOSE:
-                    node->data = node->src[0]->data;
-                    break;
-                case GGML_OP_CPY:
-                    node->data = node->src[1]->data;
-                    break;
-                default:
-                    GGML_ASSERT(!"unknown view op");
-                    break;
-            }
-        } else {
-            // see if we can reuse a parent's buffer (inplace)
-            if (ggml_op_can_inplace(node->op)) {
-                for (int i = 0; i < GGML_MAX_SRC; i++) {
-                    struct ggml_tensor * parent = node->src[i];
-                    if (parent == NULL) {
-                        break;
-                    }
-
-                    // if the node's data is external, then we cannot re-use it
-                    if ((char *) parent->data < (char *) alloc->data ||
-                        (char *) parent->data >= ((char *) alloc->data + alloc->size)) {
-                        AT_PRINTF("not reusing parent %s for %s as %p is external\n", parent->name, node->name, parent->data);
-                        continue;
-                    }
-
-                    struct hash_node * p_hn = hash_get(ht, parent);
-                    if (parent->data != NULL && p_hn->n_children == 1 && p_hn->n_views == 0 && ggml_are_same_layout(node, parent)) {
-                        if (ggml_is_view(parent)) {
-                            struct ggml_tensor * view_src = get_view_source(parent);
-                            struct hash_node * view_src_hn = hash_get(ht, view_src);
-                            if (view_src_hn->n_views == 1 && view_src_hn->n_children == 0 && view_src->data == parent->data) {
-                                // TODO: the offset of the view parent must be kept to ensure that the op doesn't overwrite
-                                // the parent's data that it will need later (same layout requirement). the problem is that then
-                                // we cannot free the tensor because the original address of the allocation is lost.
-                                // adding a view_src pointer to the tensor would solve this and simplify the code dealing with views
-                                // for now, we only reuse the parent's data if the offset is zero (view_src->data == parent->data)
-                                AT_PRINTF("reusing view parent %s (%s) for %s\n", parent->name, view_src->name, node->name);
-                                node->data = parent->data;
-                                return;
-                            }
-                        }
-                        else {
-                            AT_PRINTF("reusing parent %s for %s\n", parent->name, node->name);
-                            node->data = parent->data;
-                        }
-                        return;
-                    }
-                }
-            }
-            ggml_allocr_alloc(alloc, node);
-        }
-    }
-}
-
-static size_t ggml_allocator_alloc_graph_tensors_n(
-    struct ggml_allocr * alloc,
-    struct ggml_cgraph ** graphs, int n_graphs,
-    struct ggml_tensor *** inputs, struct ggml_tensor *** outputs) {
-
-    // reset hash table
-    struct hash_node * ht = alloc->hash_table;
-    memset(ht, 0, sizeof(struct hash_node) * GGML_GRAPH_HASHTABLE_SIZE);
-
-    // count number of children and views
-    for (int g = 0; g < n_graphs; g++) {
-        struct ggml_cgraph * gf = graphs[g];
-        for (int i = 0; i < gf->n_nodes; i++) {
-            struct ggml_tensor * node = gf->nodes[i];
-
-            if (ggml_is_view(node)) {
-                struct ggml_tensor * view_src = get_view_source(node);
-                hash_get(ht, view_src)->n_views += 1;
-            }
-
-            for (int j = 0; j < GGML_MAX_SRC; j++) {
-                struct ggml_tensor * parent = node->src[j];
-                if (parent == NULL) {
-                    break;
-                }
-                hash_get(ht, parent)->n_children += 1;
-            }
-        }
-    }
-
-    // allocate tensors
-    for (int g = 0; g < n_graphs; g++) {
-        struct ggml_cgraph * gf = graphs[g];
-        AT_PRINTF("####### graph %d/%d\n", g, n_graphs);
-        // graph inputs are allocated first to ensure that they are not overwritten by each other
-        if (inputs != NULL && inputs[g] != NULL) {
-            for (int i = 0; inputs[g][i] != NULL; i++) {
-                struct ggml_tensor * input = inputs[g][i];
-                AT_PRINTF("input: %s\n", input->name);
-                allocate_node(alloc, input);
-            }
-        }
-        for (int i = 0; i < gf->n_nodes; i++) {
-            struct ggml_tensor * node = gf->nodes[i];
-
-            // allocate parents (leafs)
-            for (int j = 0; j < GGML_MAX_SRC; j++) {
-                struct ggml_tensor * parent = node->src[j];
-                if (parent == NULL) {
-                    break;
-                }
-                allocate_node(alloc, parent);
-            }
-
-            // allocate node
-            allocate_node(alloc, node);
-
-            AT_PRINTF("exec: %s (%s) <= ", ggml_op_name(node->op), node->name);
-            for (int j = 0; j < GGML_MAX_SRC; j++) {
-                struct ggml_tensor * parent = node->src[j];
-                if (parent == NULL) {
-                    break;
-                }
-                AT_PRINTF("%s", parent->name);
-                if (j < GGML_MAX_SRC - 1 && node->src[j + 1] != NULL) {
-                    AT_PRINTF(", ");
-                }
-            }
-            AT_PRINTF("\n");
-
-            // update parents
-            for (int j = 0; j < GGML_MAX_SRC; j++) {
-                struct ggml_tensor * parent = node->src[j];
-                if (parent == NULL) {
-                    break;
-                }
-                struct hash_node * p_hn = hash_get(ht, parent);
-                p_hn->n_children -= 1;
-
-                //AT_PRINTF("parent %s: %d children, %d views\n", parent->name, parent->n_children, parent->n_views);
-
-                if (p_hn->n_children == 0 && p_hn->n_views == 0) {
-                    if (ggml_is_view(parent)) {
-                        struct ggml_tensor * view_src = get_view_source(parent);
-                        struct hash_node * view_src_hn = hash_get(ht, view_src);
-                        view_src_hn->n_views -= 1;
-                        AT_PRINTF("view_src %s: %d children, %d views\n", view_src->name, view_src->n_children, view_src->n_views);
-                        if (view_src_hn->n_views == 0 && view_src_hn->n_children == 0 && view_src->data != node->data) {
-                            ggml_allocator_free_tensor(alloc, view_src);
-                        }
-                    }
-                    else {
-                        if (parent->data != node->data) {
-                            ggml_allocator_free_tensor(alloc, parent);
-                        }
-                    }
-                }
-            }
-            AT_PRINTF("\n");
-        }
-        // free graph outputs here that wouldn't be freed otherwise because they have no children
-        if (outputs != NULL && outputs[g] != NULL) {
-            for (int i = 0; outputs[g][i] != NULL; i++) {
-                struct ggml_tensor * output = outputs[g][i];
-                AT_PRINTF("output: %s\n", output->name);
-                ggml_allocator_free_tensor(alloc, output);
-            }
-        }
-    }
-
-    return alloc->max_size;
-}
-
-size_t ggml_allocr_alloc_graph(struct ggml_allocr * alloc, struct ggml_cgraph * graph) {
-    return ggml_allocator_alloc_graph_tensors_n(alloc, &graph, 1, NULL, NULL);
-}

llm/ggml-alloc.h

@@ -1,48 +0,0 @@
-/**
- * llama.cpp - git 3ebb00935f3f0522b75df49c2769ab1774b91380
- *
- * MIT License
- *
- * Copyright (c) 2023 Georgi Gerganov
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#pragma once
-
-#include "ggml.h"
-
-#ifdef  __cplusplus
-extern "C" {
-#endif
-
-
-GGML_API struct ggml_allocr * ggml_allocr_new(void * data, size_t size, size_t alignment);
-GGML_API struct ggml_allocr * ggml_allocr_new_measure(size_t alignment);
-
-GGML_API void   ggml_allocr_free(struct ggml_allocr * alloc);
-GGML_API bool   ggml_allocr_is_measure(struct ggml_allocr * alloc);
-GGML_API void   ggml_allocr_reset(struct ggml_allocr * alloc);
-GGML_API void   ggml_allocr_alloc(struct ggml_allocr * alloc, struct ggml_tensor * tensor);
-GGML_API size_t ggml_allocr_alloc_graph(struct ggml_allocr * alloc, struct ggml_cgraph * graph);
-
-
-#ifdef  __cplusplus
-}
-#endif

llm/ggml-cuda.h

@@ -1,63 +0,0 @@
-/**
- * llama.cpp - git 3ebb00935f3f0522b75df49c2769ab1774b91380
- *
- * MIT License
- *
- * Copyright (c) 2023 Georgi Gerganov
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#pragma once
-
-#include "ggml.h"
-
-#ifdef  __cplusplus
-extern "C" {
-#endif
-
-#define GGML_CUDA_MAX_DEVICES       16
-
-void   ggml_init_cublas(void);
-void   ggml_cuda_set_tensor_split(const float * tensor_split);
-
-void   ggml_cuda_mul(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);
-bool   ggml_cuda_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);
-size_t ggml_cuda_mul_mat_get_wsize(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);
-void   ggml_cuda_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst, void * wdata, size_t wsize);
-
-// TODO: export these with GGML_API
-void * ggml_cuda_host_malloc(size_t size);
-void   ggml_cuda_host_free(void * ptr);
-
-void   ggml_cuda_transform_tensor(void * data, struct ggml_tensor * tensor);
-
-void   ggml_cuda_free_data(struct ggml_tensor * tensor);
-void   ggml_cuda_assign_buffers(struct ggml_tensor * tensor);
-void   ggml_cuda_assign_buffers_no_scratch(struct ggml_tensor * tensor);
-void   ggml_cuda_assign_buffers_force_inplace(struct ggml_tensor * tensor);
-void   ggml_cuda_set_main_device(int main_device);
-void   ggml_cuda_set_mul_mat_q(bool mul_mat_q);
-void   ggml_cuda_set_scratch_size(size_t scratch_size);
-void   ggml_cuda_free_scratch(void);
-bool   ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor);
-
-#ifdef  __cplusplus
-}
-#endif

llm/ggml-metal.h

@@ -1,106 +0,0 @@
-//go:build darwin
-
-/**
- * llama.cpp - git 3ebb00935f3f0522b75df49c2769ab1774b91380
- *
- * MIT License
- *
- * Copyright (c) 2023 Georgi Gerganov
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-// An interface allowing to compute ggml_cgraph with Metal
-//
-// This is a fully functional interface that extends ggml with GPU support for Apple devices.
-// A similar interface can be created for other GPU backends (e.g. Vulkan, CUDA, OpenCL, etc.)
-//
-// How it works?
-//
-// As long as your program can create and evaluate a ggml_cgraph on the CPU, you can use this
-// interface to evaluate the same graph on the GPU. Instead of using ggml_graph_compute(), you
-// use ggml_metal_graph_compute() (or ggml_vulkan_graph_compute(), etc.)
-//
-// You only need to make sure that all memory buffers that you used during the graph creation
-// are mapped to the device memory with the ggml_metal_add_buffer() function. This mapping is
-// used during the graph evaluation to determine the arguments of the compute kernels.
-//
-// Synchronization between device and host memory (for example for input and output tensors)
-// is done with the ggml_metal_set_tensor() and ggml_metal_get_tensor() functions.
-//
-
-#pragma once
-
-#include <stddef.h>
-#include <stdbool.h>
-
-// max memory buffers that can be mapped to the device
-#define GGML_METAL_MAX_BUFFERS 16
-
-struct ggml_tensor;
-struct ggml_cgraph;
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-struct ggml_metal_context;
-
-// number of command buffers to use
-struct ggml_metal_context * ggml_metal_init(int n_cb);
-void ggml_metal_free(struct ggml_metal_context * ctx);
-
-// set the number of command buffers to use
-void ggml_metal_set_n_cb(struct ggml_metal_context * ctx, int n_cb);
-
-// creates a mapping between a host memory buffer and a device memory buffer
-// - make sure to map all buffers used in the graph before calling ggml_metal_graph_compute
-// - the mapping is used during computation to determine the arguments of the compute kernels
-// - you don't need to keep the host memory buffer allocated as it is never accessed by Metal
-// - max_size specifies the maximum size of a tensor and is used to create shared views such
-//   that it is guaranteed that the tensor will fit in at least one of the views
-//
-bool ggml_metal_add_buffer(
-        struct ggml_metal_context * ctx,
-                       const char * name,
-                             void * data,
-                           size_t   size,
-                           size_t   max_size);
-
-// set data from host memory into the device
-void ggml_metal_set_tensor(struct ggml_metal_context * ctx, struct ggml_tensor * t);
-
-// get data from the device into host memory
-void ggml_metal_get_tensor(struct ggml_metal_context * ctx, struct ggml_tensor * t);
-
-// try to find operations that can be run concurrently in the graph
-// you should run it again if the topology of your graph changes
-void ggml_metal_graph_find_concurrency(struct ggml_metal_context * ctx, struct ggml_cgraph * gf);
-
-// if the graph has been optimized for concurrently dispatch
-bool ggml_metal_if_optimized(struct ggml_metal_context * ctx);
-
-// same as ggml_graph_compute but uses Metal
-// creates gf->n_threads command buffers in parallel
-void ggml_metal_graph_compute(struct ggml_metal_context * ctx, struct ggml_cgraph * gf);
-
-#ifdef __cplusplus
-}
-#endif
-

llm/ggml-mpi.c

@@ -1,244 +0,0 @@
-//go:build mpi
-
-/**
- * llama.cpp - git 3ebb00935f3f0522b75df49c2769ab1774b91380
- *
- * MIT License
- *
- * Copyright (c) 2023 Georgi Gerganov
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include "ggml-mpi.h"
-
-#include "ggml.h"
-
-#include <mpi.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-
-#define MIN(a, b) ((a) < (b) ? (a) : (b))
-
-#define UNUSED GGML_UNUSED
-
-struct ggml_mpi_context {
-    int rank;
-    int size;
-};
-
-void ggml_mpi_backend_init(void) {
-    MPI_Init(NULL, NULL);
-}
-
-void ggml_mpi_backend_free(void) {
-    MPI_Finalize();
-}
-
-struct ggml_mpi_context * ggml_mpi_init(void) {
-    struct ggml_mpi_context * ctx = calloc(1, sizeof(struct ggml_mpi_context));
-
-    MPI_Comm_rank(MPI_COMM_WORLD, &ctx->rank);
-    MPI_Comm_size(MPI_COMM_WORLD, &ctx->size);
-
-    return ctx;
-}
-
-void ggml_mpi_free(struct ggml_mpi_context * ctx) {
-    free(ctx);
-}
-
-int ggml_mpi_rank(struct ggml_mpi_context * ctx) {
-    return ctx->rank;
-}
-
-void ggml_mpi_eval_init(
-        struct ggml_mpi_context * ctx_mpi,
-                            int * n_tokens,
-                            int * n_past,
-                            int * n_threads) {
-    UNUSED(ctx_mpi);
-
-    // synchronize the worker node parameters with the root node
-    MPI_Barrier(MPI_COMM_WORLD);
-
-    MPI_Bcast(n_tokens,  1, MPI_INT, 0, MPI_COMM_WORLD);
-    MPI_Bcast(n_past,    1, MPI_INT, 0, MPI_COMM_WORLD);
-    MPI_Bcast(n_threads, 1, MPI_INT, 0, MPI_COMM_WORLD);
-}
-
-static int ggml_graph_get_node_idx(struct ggml_cgraph * gf, const char * name) {
-    struct ggml_tensor * t = ggml_graph_get_tensor(gf, name);
-    if (t == NULL) {
-        fprintf(stderr, "%s: tensor %s not found\n", __func__, name);
-        return -1;
-    }
-
-    for (int i = 0; i < gf->n_nodes; i++) {
-        if (gf->nodes[i] == t) {
-            return i;
-        }
-    }
-
-    fprintf(stderr, "%s: tensor %s not found in graph (should not happen)\n", __func__, name);
-    return -1;
-}
-
-static void ggml_mpi_tensor_send(struct ggml_tensor * t, int mpi_rank_dst) {
-    MPI_Datatype mpi_type;
-
-    switch (t->type) {
-        case GGML_TYPE_I32: mpi_type = MPI_INT32_T; break;
-        case GGML_TYPE_F32: mpi_type = MPI_FLOAT;   break;
-        default: GGML_ASSERT(false && "not implemented");
-    }
-
-    const int retval = MPI_Send(t->data, ggml_nelements(t), mpi_type, mpi_rank_dst, 0, MPI_COMM_WORLD);
-    GGML_ASSERT(retval == MPI_SUCCESS);
-}
-
-static void ggml_mpi_tensor_recv(struct ggml_tensor * t, int mpi_rank_src) {
-    MPI_Datatype mpi_type;
-
-    switch (t->type) {
-        case GGML_TYPE_I32: mpi_type = MPI_INT32_T; break;
-        case GGML_TYPE_F32: mpi_type = MPI_FLOAT;   break;
-        default: GGML_ASSERT(false && "not implemented");
-    }
-
-    MPI_Status status; UNUSED(status);
-
-    const int retval = MPI_Recv(t->data, ggml_nelements(t), mpi_type, mpi_rank_src, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
-    GGML_ASSERT(retval == MPI_SUCCESS);
-}
-
-// TODO: there are many improvements that can be done to this implementation
-void ggml_mpi_graph_compute_pre(
-        struct ggml_mpi_context * ctx_mpi,
-             struct ggml_cgraph * gf,
-                            int   n_layers) {
-    const int mpi_rank = ctx_mpi->rank;
-    const int mpi_size = ctx_mpi->size;
-
-    struct ggml_tensor * inp_tokens = ggml_graph_get_tensor(gf, "inp_tokens");
-    if (inp_tokens == NULL) {
-        fprintf(stderr, "%s: tensor 'inp_tokens' not found\n", __func__);
-        return;
-    }
-
-    struct ggml_tensor * inp0 = ggml_graph_get_tensor(gf, "layer_inp_0");
-    if (inp0 == NULL) {
-        fprintf(stderr, "%s: tensor 'inp0' not found\n", __func__);
-        return;
-    }
-
-    GGML_ASSERT(inp0 == gf->nodes[0]);
-
-    // distribute the compute graph into slices across the MPI nodes
-    //
-    // the main node (0) processes the last layers + the remainder of the compute graph
-    // and is responsible to pass the input tokens to the first node (1)
-    //
-    // node 1:   [(  0) * n_per_node, (  1) * n_per_node)
-    // node 2:   [(  1) * n_per_node, (  2) * n_per_node)
-    // ...
-    // node n-1: [(n-2) * n_per_node, (n-1) * n_per_node)
-    // node 0:   [(n-1) * n_per_node,            n_nodes)
-    //
-    if (mpi_rank > 0) {
-        if (mpi_rank == 1) {
-            // the first node (1) receives the input tokens from the main node (0)
-            ggml_mpi_tensor_recv(inp_tokens, 0);
-        } else {
-            // recv input data for each node into the "inp0" tensor (i.e. the first node in the compute graph)
-            ggml_mpi_tensor_recv(inp0, mpi_rank - 1);
-        }
-    } else if (mpi_size > 1) {
-        // node 0 sends the input tokens to node 1
-        ggml_mpi_tensor_send(inp_tokens, 1);
-
-        // recv the output data from the last node
-        ggml_mpi_tensor_recv(inp0, mpi_size - 1);
-    }
-
-    {
-        const int n_per_node = (n_layers + (mpi_size - 1)) / mpi_size;
-
-        const int mpi_idx = mpi_rank > 0 ? mpi_rank - 1 : mpi_size - 1;
-
-        const int il0 =               (mpi_idx + 0) * n_per_node;
-        const int il1 = MIN(n_layers, (mpi_idx + 1) * n_per_node);
-
-        char name_l0[GGML_MAX_NAME];
-        char name_l1[GGML_MAX_NAME];
-
-        snprintf(name_l0, sizeof(name_l0), "layer_inp_%d", il0);
-        snprintf(name_l1, sizeof(name_l1), "layer_inp_%d", il1);
-
-        const int idx_l0 =                ggml_graph_get_node_idx(gf, name_l0);
-        const int idx_l1 = mpi_rank > 0 ? ggml_graph_get_node_idx(gf, name_l1) + 1 : gf->n_nodes;
-
-        if (idx_l0 < 0 || idx_l1 < 0) {
-            fprintf(stderr, "%s: layer input nodes not found\n", __func__);
-            return;
-        }
-
-        // attach the input data to all nodes that need it
-        // TODO: not great - should be able to do this without modifying the compute graph (see next TODO below)
-        for (int i = idx_l0; i < idx_l1; i++) {
-            if (gf->nodes[i]->src[0] == gf->nodes[idx_l0]) {
-                gf->nodes[i]->src[0] =  inp0;
-            }
-            if (gf->nodes[i]->src[1] == gf->nodes[idx_l0]) {
-                gf->nodes[i]->src[1] =  inp0;
-            }
-        }
-
-        // TODO: instead of rearranging the nodes, we should be able to execute a subset of the compute graph
-        for (int i = 1; i < idx_l1 - idx_l0; i++) {
-            gf->nodes[i] = gf->nodes[idx_l0 + i];
-            gf->grads[i] = gf->grads[idx_l0 + i];
-        }
-
-        // the first node performs the "get_rows" operation, the rest of the nodes get the data from the previous node
-        if (mpi_idx != 0) {
-            gf->nodes[0]->op = GGML_OP_NONE;
-        }
-
-        gf->n_nodes = idx_l1 - idx_l0;
-
-        //fprintf(stderr, "%s: node %d: processing %d nodes [%d, %d)\n", __func__, mpi_rank, gf->n_nodes, il0, il1);
-    }
-}
-
-void ggml_mpi_graph_compute_post(
-        struct ggml_mpi_context * ctx_mpi,
-             struct ggml_cgraph * gf,
-                            int   n_layers) {
-    UNUSED(n_layers);
-
-    const int mpi_rank = ctx_mpi->rank;
-    const int mpi_size = ctx_mpi->size;
-
-    // send the output data to the next node
-    if (mpi_rank > 0) {
-        ggml_mpi_tensor_send(gf->nodes[gf->n_nodes - 1], (mpi_rank + 1) % mpi_size);
-    }
-}

llm/ggml-mpi.h

@@ -1,67 +0,0 @@
-//go:build mpi
-
-/**
- * llama.cpp - git 3ebb00935f3f0522b75df49c2769ab1774b91380
- *
- * MIT License
- *
- * Copyright (c) 2023 Georgi Gerganov
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#pragma once
-
-struct ggml_context;
-struct ggml_tensor;
-struct ggml_cgraph;
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-struct ggml_mpi_context;
-
-void ggml_mpi_backend_init(void);
-void ggml_mpi_backend_free(void);
-
-struct ggml_mpi_context * ggml_mpi_init(void);
-void ggml_mpi_free(struct ggml_mpi_context * ctx);
-
-int ggml_mpi_rank(struct ggml_mpi_context * ctx);
-
-void ggml_mpi_eval_init(
-        struct ggml_mpi_context * ctx_mpi,
-                            int * n_tokens,
-                            int * n_past,
-                            int * n_threads);
-
-void ggml_mpi_graph_compute_pre(
-        struct ggml_mpi_context * ctx_mpi,
-             struct ggml_cgraph * gf,
-                            int   n_layers);
-
-void ggml_mpi_graph_compute_post(
-        struct ggml_mpi_context * ctx_mpi,
-             struct ggml_cgraph * gf,
-                            int   n_layers);
-
-#ifdef __cplusplus
-}
-#endif

llm/ggml-opencl.h

@@ -1,53 +0,0 @@
-//go:build opencl
-
-/**
- * llama.cpp - git 3ebb00935f3f0522b75df49c2769ab1774b91380
- *
- * MIT License
- *
- * Copyright (c) 2023 Georgi Gerganov
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#pragma once
-
-#include "ggml.h"
-
-#ifdef  __cplusplus
-extern "C" {
-#endif
-
-void ggml_cl_init(void);
-
-void   ggml_cl_mul(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);
-bool   ggml_cl_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);
-size_t ggml_cl_mul_mat_get_wsize(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);
-void   ggml_cl_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst, void * wdata, size_t wsize);
-
-void * ggml_cl_host_malloc(size_t size);
-void   ggml_cl_host_free(void * ptr);
-
-void ggml_cl_free_data(const struct ggml_tensor* tensor);
-
-void ggml_cl_transform_tensor(void * data, struct ggml_tensor * tensor);
-
-#ifdef  __cplusplus
-}
-#endif

llm/ggml_llama.go

@@ -0,0 +1,687 @@
+package llm
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"embed"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"log"
+	"math/rand"
+	"net/http"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/jmorganca/ollama/api"
+)
+
+const ModelFamilyLlama ModelFamily = "llama"
+
+//go:embed llama.cpp/ggml/build/*/bin/*
+var llamaCppEmbed embed.FS
+
+var (
+	ggmlGPU = path.Join("llama.cpp", "ggml", "build", "gpu", "bin")
+	ggmlCPU = path.Join("llama.cpp", "ggml", "build", "cpu", "bin")
+)
+
+var (
+	ggmlInit       sync.Once
+	ggmlRunnerPath string
+)
+
+func osPath(llamaPath string) string {
+	if runtime.GOOS == "windows" {
+		return path.Join(llamaPath, "Release")
+	}
+
+	return llamaPath
+}
+
+func initGGML() {
+	ggmlInit.Do(func() {
+		tmpDir, err := os.MkdirTemp("", "llama-*")
+		if err != nil {
+			log.Fatalf("llama.cpp: failed to create temp dir: %v", err)
+		}
+
+		llamaPath := osPath(ggmlGPU)
+		if _, err := fs.Stat(llamaCppEmbed, llamaPath); err != nil {
+			llamaPath = osPath(ggmlCPU)
+			if _, err := fs.Stat(llamaCppEmbed, llamaPath); err != nil {
+				log.Fatalf("llama.cpp executable not found")
+			}
+		}
+
+		files := []string{"server"}
+		switch runtime.GOOS {
+		case "windows":
+			files = []string{"server.exe"}
+		case "darwin":
+			if llamaPath == osPath(ggmlGPU) {
+				files = append(files, "ggml-metal.metal")
+			}
+		}
+
+		for _, f := range files {
+			srcPath := path.Join(llamaPath, f)
+			destPath := filepath.Join(tmpDir, f)
+
+			srcFile, err := llamaCppEmbed.Open(srcPath)
+			if err != nil {
+				log.Fatalf("read llama.cpp %s: %v", f, err)
+			}
+			defer srcFile.Close()
+
+			destFile, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o755)
+			if err != nil {
+				log.Fatalf("write llama.cpp %s: %v", f, err)
+			}
+			defer destFile.Close()
+
+			if _, err := io.Copy(destFile, srcFile); err != nil {
+				log.Fatalf("copy llama.cpp %s: %v", f, err)
+			}
+		}
+
+		ggmlRunnerPath = filepath.Join(tmpDir, "server")
+		if runtime.GOOS == "windows" {
+			ggmlRunnerPath = filepath.Join(tmpDir, "server.exe")
+		}
+	})
+}
+
+type ModelRunner struct {
+	Path string // path to the model runner executable
+}
+
+func ggmlRunner() ModelRunner {
+	initGGML()
+	return ModelRunner{Path: ggmlRunnerPath}
+}
+
+type llamaModel struct {
+	hyperparameters llamaHyperparameters
+}
+
+func (llm *llamaModel) ModelFamily() ModelFamily {
+	return ModelFamilyLlama
+}
+
+func (llm *llamaModel) ModelType() ModelType {
+	switch llm.hyperparameters.NumLayer {
+	case 26:
+		return ModelType3B
+	case 32:
+		return ModelType7B
+	case 40:
+		return ModelType13B
+	case 48:
+		return ModelType34B
+	case 60:
+		return ModelType30B
+	case 80:
+		return ModelType65B
+	}
+
+	// TODO: find a better default
+	return ModelType7B
+}
+
+func (llm *llamaModel) FileType() FileType {
+	return llm.hyperparameters.FileType
+}
+
+type llamaHyperparameters struct {
+	// NumVocab is the size of the model's vocabulary.
+	NumVocab uint32
+
+	// NumEmbd is the size of the model's embedding layer.
+	NumEmbd uint32
+	NumMult uint32
+	NumHead uint32
+
+	// NumLayer is the number of layers in the model.
+	NumLayer uint32
+	NumRot   uint32
+
+	// FileType describes the quantization level of the model, e.g. Q4_0, Q5_K, etc.
+	FileType llamaFileType
+}
+
+type llamaFileType uint32
+
+const (
+	llamaFileTypeF32 llamaFileType = iota
+	llamaFileTypeF16
+	llamaFileTypeQ4_0
+	llamaFileTypeQ4_1
+	llamaFileTypeQ4_1_F16
+	llamaFileTypeQ8_0 llamaFileType = iota + 2
+	llamaFileTypeQ5_0
+	llamaFileTypeQ5_1
+	llamaFileTypeQ2_K
+	llamaFileTypeQ3_K_S
+	llamaFileTypeQ3_K_M
+	llamaFileTypeQ3_K_L
+	llamaFileTypeQ4_K_S
+	llamaFileTypeQ4_K_M
+	llamaFileTypeQ5_K_S
+	llamaFileTypeQ5_K_M
+	llamaFileTypeQ6_K
+)
+
+func (ft llamaFileType) String() string {
+	switch ft {
+	case llamaFileTypeF32:
+		return "F32"
+	case llamaFileTypeF16:
+		return "F16"
+	case llamaFileTypeQ4_0:
+		return "Q4_0"
+	case llamaFileTypeQ4_1:
+		return "Q4_1"
+	case llamaFileTypeQ4_1_F16:
+		return "Q4_1_F16"
+	case llamaFileTypeQ8_0:
+		return "Q8_0"
+	case llamaFileTypeQ5_0:
+		return "Q5_0"
+	case llamaFileTypeQ5_1:
+		return "Q5_1"
+	case llamaFileTypeQ2_K:
+		return "Q2_K"
+	case llamaFileTypeQ3_K_S:
+		return "Q3_K_S"
+	case llamaFileTypeQ3_K_M:
+		return "Q3_K_M"
+	case llamaFileTypeQ3_K_L:
+		return "Q3_K_L"
+	case llamaFileTypeQ4_K_S:
+		return "Q4_K_S"
+	case llamaFileTypeQ4_K_M:
+		return "Q4_K_M"
+	case llamaFileTypeQ5_K_S:
+		return "Q5_K_S"
+	case llamaFileTypeQ5_K_M:
+		return "Q5_K_M"
+	case llamaFileTypeQ6_K:
+		return "Q6_K"
+	default:
+		return "Unknown"
+	}
+}
+
+type Running struct {
+	Port   int
+	Cmd    *exec.Cmd
+	Cancel context.CancelFunc
+}
+
+type llama struct {
+	api.Options
+	Running
+}
+
+func newLlama(model string, adapters []string, runner ModelRunner, opts api.Options) (*llama, error) {
+	if _, err := os.Stat(model); err != nil {
+		return nil, err
+	}
+
+	if _, err := os.Stat(runner.Path); err != nil {
+		return nil, err
+	}
+
+	if len(adapters) > 1 {
+		return nil, errors.New("ollama supports only one lora adapter, but multiple were provided")
+	}
+
+	params := []string{
+		"--model", model,
+		"--ctx-size", fmt.Sprintf("%d", opts.NumCtx),
+		"--gqa", fmt.Sprintf("%d", opts.NumGQA),
+		"--rope-freq-base", fmt.Sprintf("%f", opts.RopeFrequencyBase),
+		"--rope-freq-scale", fmt.Sprintf("%f", opts.RopeFrequencyScale),
+		"--batch-size", fmt.Sprintf("%d", opts.NumBatch),
+		"--n-gpu-layers", fmt.Sprintf("%d", opts.NumGPU),
+		"--embedding",
+	}
+
+	if len(adapters) > 0 {
+		// TODO: applying multiple adapters is not supported by the llama.cpp server yet
+		params = append(params, "--lora", adapters[0])
+	}
+
+	if opts.NumThread > 0 {
+		params = append(params, "--threads", fmt.Sprintf("%d", opts.NumThread))
+	}
+
+	if !opts.F16KV {
+		params = append(params, "--memory-f32")
+	}
+	if opts.UseMLock {
+		params = append(params, "--mlock")
+	}
+	if !opts.UseMMap {
+		params = append(params, "--no-mmap")
+	}
+	if opts.UseNUMA {
+		params = append(params, "--numa")
+	}
+
+	// start the llama.cpp server with a retry in case the port is already in use
+	for try := 0; try < 3; try++ {
+		port := rand.Intn(65535-49152) + 49152 // get a random port in the ephemeral range
+		ctx, cancel := context.WithCancel(context.Background())
+		cmd := exec.CommandContext(
+			ctx,
+			runner.Path,
+			append(params, "--port", strconv.Itoa(port))...,
+		)
+		cmd.Stdout = os.Stderr
+		cmd.Stderr = os.Stderr
+
+		llm := &llama{Options: opts, Running: Running{Port: port, Cmd: cmd, Cancel: cancel}}
+
+		if err := waitForServer(llm); err != nil {
+			log.Printf("error starting llama.cpp server: %v", err)
+			llm.Close()
+			// try again
+			continue
+		}
+		// server started successfully
+		return llm, nil
+	}
+
+	return nil, fmt.Errorf("max retry exceeded starting llama.cpp")
+}
+
+func waitForServer(llm *llama) error {
+	log.Print("starting llama.cpp server")
+	var stderr bytes.Buffer
+	llm.Cmd.Stderr = &stderr
+	err := llm.Cmd.Start()
+	if err != nil {
+		return fmt.Errorf("error starting the external llama.cpp server: %w", err)
+	}
+
+	exitChan := make(chan error, 1)
+
+	// the server is a long running process, watch for it exiting to keep track of something going wrong
+	go func() {
+		err := llm.Cmd.Wait()
+		log.Print(stderr.String())
+		exitChan <- err
+	}()
+
+	// wait for the server to start responding
+	start := time.Now()
+	expiresAt := time.Now().Add(30 * time.Second)
+	ticker := time.NewTicker(100 * time.Millisecond)
+
+	log.Print("waiting for llama.cpp server to start responding")
+
+	for {
+		select {
+		case <-ticker.C:
+			if time.Now().After(expiresAt) {
+				return fmt.Errorf("llama.cpp server did not start responding within 30 seconds, retrying")
+			}
+			if err := llm.Ping(context.Background()); err == nil {
+				log.Printf("llama.cpp server started in %f seconds", time.Since(start).Seconds())
+				return nil
+			}
+		case err := <-exitChan:
+			return fmt.Errorf("llama.cpp server exited unexpectedly: %w", err)
+		}
+	}
+}
+
+func (llm *llama) Close() {
+	llm.Running.Cmd.Cancel()
+}
+
+func (llm *llama) SetOptions(opts api.Options) {
+	llm.Options = opts
+}
+
+type GenerationSettings struct {
+	FrequencyPenalty float64       `json:"frequency_penalty"`
+	IgnoreEOS        bool          `json:"ignore_eos"`
+	LogitBias        []interface{} `json:"logit_bias"`
+	Mirostat         int           `json:"mirostat"`
+	MirostatEta      float64       `json:"mirostat_eta"`
+	MirostatTau      float64       `json:"mirostat_tau"`
+	Model            string        `json:"model"`
+	NCtx             int           `json:"n_ctx"`
+	NKeep            int           `json:"n_keep"`
+	NPredict         int           `json:"n_predict"`
+	NProbs           int           `json:"n_probs"`
+	PenalizeNl       bool          `json:"penalize_nl"`
+	PresencePenalty  float64       `json:"presence_penalty"`
+	RepeatLastN      int           `json:"repeat_last_n"`
+	RepeatPenalty    float64       `json:"repeat_penalty"`
+	Seed             uint32        `json:"seed"`
+	Stop             []string      `json:"stop"`
+	Stream           bool          `json:"stream"`
+	Temp             float64       `json:"temp"`
+	TfsZ             float64       `json:"tfs_z"`
+	TopK             int           `json:"top_k"`
+	TopP             float64       `json:"top_p"`
+	TypicalP         float64       `json:"typical_p"`
+}
+
+type Timings struct {
+	PredictedN  int     `json:"predicted_n"`
+	PredictedMS float64 `json:"predicted_ms"`
+	PromptN     int     `json:"prompt_n"`
+	PromptMS    float64 `json:"prompt_ms"`
+}
+
+type Prediction struct {
+	Content string `json:"content"`
+	Model   string `json:"model"`
+	Prompt  string `json:"prompt"`
+	Stop    bool   `json:"stop"`
+
+	Timings `json:"timings"`
+}
+
+type PredictRequest struct {
+	Stream           bool            `json:"stream"`
+	NPredict         int             `json:"n_predict,omitempty"`
+	TopK             int             `json:"top_k,omitempty"`
+	TopP             float32         `json:"top_p,omitempty"`
+	TfsZ             float32         `json:"tfs_z,omitempty"`
+	TypicalP         float32         `json:"typical_p,omitempty"`
+	RepeatLastN      int             `json:"repeat_last_n,omitempty"`
+	Temperature      float32         `json:"temperature,omitempty"`
+	RepeatPenalty    float32         `json:"repeat_penalty,omitempty"`
+	PresencePenalty  float32         `json:"presence_penalty,omitempty"`
+	FrequencyPenalty float32         `json:"frequency_penalty,omitempty"`
+	Mirostat         int             `json:"mirostat,omitempty"`
+	MirostatTau      float32         `json:"mirostat_tau,omitempty"`
+	MirostatEta      float32         `json:"mirostat_eta,omitempty"`
+	PenalizeNl       bool            `json:"penalize_nl,omitempty"`
+	NKeep            int             `json:"n_keep,omitempty"`
+	Seed             int             `json:"seed,omitempty"`
+	Prompt           string          `json:"prompt,omitempty"`
+	NProbs           int             `json:"n_probs,omitempty"`
+	LogitBias        map[int]float32 `json:"logit_bias,omitempty"`
+	IgnoreEos        bool            `json:"ignore_eos,omitempty"`
+	Stop             []string        `json:"stop,omitempty"`
+}
+
+func (llm *llama) Predict(ctx context.Context, prevContext []int, prompt string, fn func(api.GenerateResponse)) error {
+	prevConvo, err := llm.Decode(ctx, prevContext)
+	if err != nil {
+		return err
+	}
+
+	var nextContext strings.Builder
+	nextContext.WriteString(prevConvo)
+	nextContext.WriteString(prompt)
+
+	endpoint := fmt.Sprintf("http://127.0.0.1:%d/completion", llm.Port)
+	predReq := PredictRequest{
+		Prompt:           nextContext.String(),
+		Stream:           true,
+		NPredict:         llm.NumPredict,
+		NKeep:            llm.NumKeep,
+		Temperature:      llm.Temperature,
+		TopK:             llm.TopK,
+		TopP:             llm.TopP,
+		TfsZ:             llm.TFSZ,
+		TypicalP:         llm.TypicalP,
+		RepeatLastN:      llm.RepeatLastN,
+		RepeatPenalty:    llm.RepeatPenalty,
+		PresencePenalty:  llm.PresencePenalty,
+		FrequencyPenalty: llm.FrequencyPenalty,
+		Mirostat:         llm.Mirostat,
+		MirostatTau:      llm.MirostatTau,
+		MirostatEta:      llm.MirostatEta,
+		PenalizeNl:       llm.PenalizeNewline,
+		Stop:             llm.Stop,
+	}
+	data, err := json.Marshal(predReq)
+	if err != nil {
+		return fmt.Errorf("error marshaling data: %v", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewBuffer(data))
+	if err != nil {
+		return fmt.Errorf("error creating POST request: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("POST predict: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode >= 400 {
+		bodyBytes, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return fmt.Errorf("failed reading llm error response: %w", err)
+		}
+		log.Printf("llm predict error: %s", bodyBytes)
+		return fmt.Errorf("%s", bodyBytes)
+	}
+
+	scanner := bufio.NewScanner(resp.Body)
+	for scanner.Scan() {
+		select {
+		case <-ctx.Done():
+			// This handles the request cancellation
+			return ctx.Err()
+		default:
+			line := scanner.Text()
+			if line == "" {
+				continue
+			}
+
+			// Read data from the server-side event stream
+			if strings.HasPrefix(line, "data: ") {
+				evt := line[6:]
+				var p Prediction
+				if err := json.Unmarshal([]byte(evt), &p); err != nil {
+					return fmt.Errorf("error unmarshaling llm prediction response: %v", err)
+				}
+
+				if p.Content != "" {
+					fn(api.GenerateResponse{Response: p.Content})
+					nextContext.WriteString(p.Content)
+				}
+
+				if p.Stop {
+					embd, err := llm.Encode(ctx, nextContext.String())
+					if err != nil {
+						return fmt.Errorf("encoding context: %v", err)
+					}
+
+					fn(api.GenerateResponse{
+						Done:               true,
+						Context:            embd,
+						PromptEvalCount:    p.PromptN,
+						PromptEvalDuration: parseDurationMs(p.PromptMS),
+						EvalCount:          p.PredictedN,
+						EvalDuration:       parseDurationMs(p.PredictedMS),
+					})
+
+					return nil
+				}
+			}
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return fmt.Errorf("error reading llm response: %v", err)
+	}
+
+	return nil
+}
+
+type TokenizeRequest struct {
+	Content string `json:"content"`
+}
+
+type TokenizeResponse struct {
+	Tokens []int `json:"tokens"`
+}
+
+func (llm *llama) Encode(ctx context.Context, prompt string) ([]int, error) {
+	endpoint := fmt.Sprintf("http://127.0.0.1:%d/tokenize", llm.Port)
+	data, err := json.Marshal(TokenizeRequest{Content: prompt})
+	if err != nil {
+		return nil, fmt.Errorf("marshaling encode data: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewBuffer(data))
+	if err != nil {
+		return nil, fmt.Errorf("encode request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("do encode request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("read encode request: %w", err)
+	}
+
+	if resp.StatusCode >= 400 {
+		log.Printf("llm encode error: %s", body)
+		return nil, fmt.Errorf("%s", body)
+	}
+
+	var encoded TokenizeResponse
+	if err := json.Unmarshal(body, &encoded); err != nil {
+		return nil, fmt.Errorf("unmarshal encode response: %w", err)
+	}
+
+	return encoded.Tokens, nil
+}
+
+type DetokenizeRequest struct {
+	Tokens []int `json:"tokens"`
+}
+
+type DetokenizeResponse struct {
+	Content string `json:"content"`
+}
+
+func (llm *llama) Decode(ctx context.Context, tokens []int) (string, error) {
+	if len(tokens) == 0 {
+		return "", nil
+	}
+	endpoint := fmt.Sprintf("http://127.0.0.1:%d/detokenize", llm.Port)
+	data, err := json.Marshal(DetokenizeRequest{Tokens: tokens})
+	if err != nil {
+		return "", fmt.Errorf("marshaling decode data: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewBuffer(data))
+	if err != nil {
+		return "", fmt.Errorf("decode request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("do decode request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("read decode request: %w", err)
+	}
+
+	if resp.StatusCode >= 400 {
+		log.Printf("llm decode error: %s", body)
+		return "", fmt.Errorf("%s", body)
+	}
+
+	var decoded DetokenizeResponse
+	if err := json.Unmarshal(body, &decoded); err != nil {
+		return "", fmt.Errorf("unmarshal encode response: %w", err)
+	}
+
+	// decoded content contains a leading whitespace
+	decoded.Content, _ = strings.CutPrefix(decoded.Content, "")
+
+	return decoded.Content, nil
+}
+
+type EmbeddingRequest struct {
+	Content string `json:"content"`
+}
+
+type EmbeddingResponse struct {
+	Embedding []float64 `json:"embedding"`
+}
+
+func (llm *llama) Embedding(ctx context.Context, input string) ([]float64, error) {
+	endpoint := fmt.Sprintf("http://127.0.0.1:%d/embedding", llm.Port)
+	data, err := json.Marshal(TokenizeRequest{Content: input})
+	if err != nil {
+		return nil, fmt.Errorf("error marshaling embed data: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewBuffer(data))
+	if err != nil {
+		return nil, fmt.Errorf("error creating embed request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("POST embedding: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("error reading embed response: %w", err)
+	}
+
+	if resp.StatusCode >= 400 {
+		log.Printf("llm encode error: %s", body)
+		return nil, fmt.Errorf("%s", body)
+	}
+
+	var embedding EmbeddingResponse
+	if err := json.Unmarshal(body, &embedding); err != nil {
+		return nil, fmt.Errorf("unmarshal tokenize response: %w", err)
+	}
+
+	return embedding.Embedding, nil
+}
+
+// Ping checks that the server subprocess is still running and responding to requests
+func (llm *llama) Ping(ctx context.Context) error {
+	resp, err := http.Head(fmt.Sprintf("http://127.0.0.1:%d", llm.Running.Port))
+	if err != nil {
+		return fmt.Errorf("ping resp: %w", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("unexpected ping status: %s", resp.Status)
+	}
+	return nil
+}

llm/k_quants.h

@@ -1,191 +0,0 @@
-/**
- * llama.cpp - git 3ebb00935f3f0522b75df49c2769ab1774b91380
- *
- * MIT License
- *
- * Copyright (c) 2023 Georgi Gerganov
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#pragma once
-
-#include "ggml.h"
-
-#include <stdint.h>
-#include <assert.h>
-#include <stddef.h>
-
-// Super-block size
-#ifdef GGML_QKK_64
-#define QK_K 64
-#define K_SCALE_SIZE 4
-#else
-#define QK_K 256
-#define K_SCALE_SIZE 12
-#endif
-
-#ifndef static_assert
-#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201100L)
-#define static_assert(cond, msg) _Static_assert(cond, msg)
-#else
-#define static_assert(cond, msg) struct global_scope_noop_trick
-#endif
-#endif
-
-//
-// Super-block quantization structures
-//
-
-// 2-bit quantization
-// weight is represented as x = a * q + b
-// 16 blocks of 16 elemenets each
-// Effectively 2.5625 bits per weight
-typedef struct {
-    uint8_t scales[QK_K/16]; // scales and mins, quantized with 4 bits
-    uint8_t qs[QK_K/4];      // quants
-    ggml_fp16_t d;           // super-block scale for quantized scales
-    ggml_fp16_t dmin;        // super-block scale for quantized mins
-} block_q2_K;
-static_assert(sizeof(block_q2_K) == 2*sizeof(ggml_fp16_t) + QK_K/16 + QK_K/4, "wrong q2_K block size/padding");
-
-// 3-bit quantization
-// weight is represented as x = a * q
-// 16 blocks of 16 elemenets each
-// Effectively 3.4375 bits per weight
-#ifdef GGML_QKK_64
-typedef struct {
-    uint8_t hmask[QK_K/8];     // quants - high bit
-    uint8_t qs[QK_K/4];        // quants - low 2 bits
-    uint8_t scales[2];
-    ggml_fp16_t d;             // super-block scale
-} block_q3_K;
-static_assert(sizeof(block_q3_K) == sizeof(ggml_fp16_t) + QK_K / 4 + QK_K / 8 + 2, "wrong q3_K block size/padding");
-#else
-typedef struct {
-    uint8_t hmask[QK_K/8];     // quants - high bit
-    uint8_t qs[QK_K/4];        // quants - low 2 bits
-    uint8_t scales[12];        // scales, quantized with 6 bits
-    ggml_fp16_t d;             // super-block scale
-} block_q3_K;
-static_assert(sizeof(block_q3_K) == sizeof(ggml_fp16_t) + QK_K / 4 + QK_K / 8 + 12, "wrong q3_K block size/padding");
-#endif
-
-// 4-bit quantization
-// 16 blocks of 32 elements each
-// weight is represented as x = a * q + b
-// Effectively 4.5 bits per weight
-#ifdef GGML_QKK_64
-typedef struct {
-    ggml_fp16_t d[2];          // super-block scales/mins
-    uint8_t scales[2];         // 4-bit block scales/mins
-    uint8_t qs[QK_K/2];        // 4--bit quants
-} block_q4_K;
-static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_fp16_t) + QK_K/2 + 2, "wrong q4_K block size/padding");
-#else
-typedef struct {
-    ggml_fp16_t d;             // super-block scale for quantized scales
-    ggml_fp16_t dmin;          // super-block scale for quantized mins
-    uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits
-    uint8_t qs[QK_K/2];        // 4--bit quants
-} block_q4_K;
-static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_fp16_t) + K_SCALE_SIZE + QK_K/2, "wrong q4_K block size/padding");
-#endif
-
-// 5-bit quantization
-// 16 blocks of 32 elements each
-// weight is represented as x = a * q + b
-// Effectively 5.5 bits per weight
-#ifdef GGML_QKK_64
-typedef struct {
-    ggml_fp16_t d;               // super-block scale
-    int8_t  scales[QK_K/16];     // 8-bit block scales
-    uint8_t qh[QK_K/8];          // quants, high bit
-    uint8_t qs[QK_K/2];          // quants, low 4 bits
-} block_q5_K;
-static_assert(sizeof(block_q5_K) == sizeof(ggml_fp16_t) + QK_K/2 + QK_K/8 + QK_K/16, "wrong q5_K block size/padding");
-#else
-typedef struct {
-    ggml_fp16_t d;               // super-block scale for quantized scales
-    ggml_fp16_t dmin;            // super-block scale for quantized mins
-    uint8_t scales[K_SCALE_SIZE];   // scales and mins, quantized with 6 bits
-    uint8_t qh[QK_K/8];          // quants, high bit
-    uint8_t qs[QK_K/2];          // quants, low 4 bits
-} block_q5_K;
-static_assert(sizeof(block_q5_K) == 2*sizeof(ggml_fp16_t) + K_SCALE_SIZE + QK_K/2 + QK_K/8, "wrong q5_K block size/padding");
-#endif
-
-// 6-bit quantization
-// weight is represented as x = a * q
-// 16 blocks of 16 elemenets each
-// Effectively 6.5625 bits per weight
-typedef struct {
-    uint8_t ql[QK_K/2];      // quants, lower 4 bits
-    uint8_t qh[QK_K/4];      // quants, upper 2 bits
-    int8_t  scales[QK_K/16]; // scales, quantized with 8 bits
-    ggml_fp16_t d;           // super-block scale
-} block_q6_K;
-static_assert(sizeof(block_q6_K) == sizeof(ggml_fp16_t) + QK_K / 16 + 3*QK_K/4, "wrong q6_K block size/padding");
-
-// This is only used for intermediate quantization and dot products
-typedef struct {
-    float   d;              // delta
-    int8_t  qs[QK_K];       // quants
-    int16_t bsums[QK_K/16]; // sum of quants in groups of 16
-} block_q8_K;
-static_assert(sizeof(block_q8_K) == sizeof(float) + QK_K + QK_K/16*sizeof(int16_t), "wrong q8_K block size/padding");
-
-
-// Quantization
-void quantize_row_q2_K_reference(const float * restrict x, block_q2_K * restrict y, int k);
-void quantize_row_q3_K_reference(const float * restrict x, block_q3_K * restrict y, int k);
-void quantize_row_q4_K_reference(const float * restrict x, block_q4_K * restrict y, int k);
-void quantize_row_q5_K_reference(const float * restrict x, block_q5_K * restrict y, int k);
-void quantize_row_q6_K_reference(const float * restrict x, block_q6_K * restrict y, int k);
-void quantize_row_q8_K_reference(const float * restrict x, block_q8_K * restrict y, int k);
-
-void quantize_row_q2_K(const float * restrict x, void * restrict y, int k);
-void quantize_row_q3_K(const float * restrict x, void * restrict y, int k);
-void quantize_row_q4_K(const float * restrict x, void * restrict y, int k);
-void quantize_row_q5_K(const float * restrict x, void * restrict y, int k);
-void quantize_row_q6_K(const float * restrict x, void * restrict y, int k);
-void quantize_row_q8_K(const float * restrict x, void * restrict y, int k);
-
-// Dequantization
-void dequantize_row_q2_K(const block_q2_K * restrict x, float * restrict y, int k);
-void dequantize_row_q3_K(const block_q3_K * restrict x, float * restrict y, int k);
-void dequantize_row_q4_K(const block_q4_K * restrict x, float * restrict y, int k);
-void dequantize_row_q5_K(const block_q5_K * restrict x, float * restrict y, int k);
-void dequantize_row_q6_K(const block_q6_K * restrict x, float * restrict y, int k);
-void dequantize_row_q8_K(const block_q8_K * restrict x, float * restrict y, int k);
-
-// Dot product
-void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);
-void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);
-void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);
-void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);
-void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);
-
-// Quantization with histogram collection
-size_t ggml_quantize_q2_K(const float * src, void * dst, int n, int k, int64_t * hist);
-size_t ggml_quantize_q3_K(const float * src, void * dst, int n, int k, int64_t * hist);
-size_t ggml_quantize_q4_K(const float * src, void * dst, int n, int k, int64_t * hist);
-size_t ggml_quantize_q5_K(const float * src, void * dst, int n, int k, int64_t * hist);
-size_t ggml_quantize_q6_K(const float * src, void * dst, int n, int k, int64_t * hist);
-

llm/llama-util.h

@@ -1,579 +0,0 @@
-/**
- * llama.cpp - git 3ebb00935f3f0522b75df49c2769ab1774b91380
- *
- * MIT License
- *
- * Copyright (c) 2023 Georgi Gerganov
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-// Internal header to be included only by llama.cpp.
-// Contains wrappers around OS interfaces.
-
-#ifndef LLAMA_UTIL_H
-#define LLAMA_UTIL_H
-
-#include <cstdio>
-#include <cstdint>
-#include <cerrno>
-#include <cstring>
-#include <cstdarg>
-#include <cstdlib>
-#include <climits>
-
-#include <string>
-#include <vector>
-#include <stdexcept>
-
-#ifdef __has_include
-    #if __has_include(<unistd.h>)
-        #include <unistd.h>
-        #if defined(_POSIX_MAPPED_FILES)
-            #include <sys/mman.h>
-        #endif
-        #if defined(_POSIX_MEMLOCK_RANGE)
-            #include <sys/resource.h>
-        #endif
-    #endif
-#endif
-
-#if defined(_WIN32)
-    #define WIN32_LEAN_AND_MEAN
-    #ifndef NOMINMAX
-        #define NOMINMAX
-    #endif
-    #include <windows.h>
-    #include <io.h>
-    #include <stdio.h> // for _fseeki64
-#endif
-
-#define LLAMA_ASSERT(x) \
-    do { \
-        if (!(x)) { \
-            fprintf(stderr, "LLAMA_ASSERT: %s:%d: %s\n", __FILE__, __LINE__, #x); \
-            abort(); \
-        } \
-    } while (0)
-
-#ifdef __GNUC__
-#ifdef __MINGW32__
-__attribute__((format(gnu_printf, 1, 2)))
-#else
-__attribute__((format(printf, 1, 2)))
-#endif
-#endif
-static std::string format(const char * fmt, ...) {
-    va_list ap, ap2;
-    va_start(ap, fmt);
-    va_copy(ap2, ap);
-    int size = vsnprintf(NULL, 0, fmt, ap);
-    LLAMA_ASSERT(size >= 0 && size < INT_MAX);
-    std::vector<char> buf(size + 1);
-    int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2);
-    LLAMA_ASSERT(size2 == size);
-    va_end(ap2);
-    va_end(ap);
-    return std::string(buf.data(), size);
-}
-
-struct llama_file {
-    // use FILE * so we don't have to re-open the file to mmap
-    FILE * fp;
-    size_t size;
-
-    llama_file(const char * fname, const char * mode) {
-        fp = std::fopen(fname, mode);
-        if (fp == NULL) {
-            throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));
-        }
-        seek(0, SEEK_END);
-        size = tell();
-        seek(0, SEEK_SET);
-    }
-
-    size_t tell() const {
-#ifdef _WIN32
-        __int64 ret = _ftelli64(fp);
-#else
-        long ret = std::ftell(fp);
-#endif
-        LLAMA_ASSERT(ret != -1); // this really shouldn't fail
-        return (size_t) ret;
-    }
-
-    void seek(size_t offset, int whence) {
-#ifdef _WIN32
-        int ret = _fseeki64(fp, (__int64) offset, whence);
-#else
-        int ret = std::fseek(fp, (long) offset, whence);
-#endif
-        LLAMA_ASSERT(ret == 0); // same
-    }
-
-    void read_raw(void * ptr, size_t len) const {
-        if (len == 0) {
-            return;
-        }
-        errno = 0;
-        std::size_t ret = std::fread(ptr, len, 1, fp);
-        if (ferror(fp)) {
-            throw std::runtime_error(format("read error: %s", strerror(errno)));
-        }
-        if (ret != 1) {
-            throw std::runtime_error(std::string("unexpectedly reached end of file"));
-        }
-    }
-
-    std::uint32_t read_u32() {
-        std::uint32_t ret;
-        read_raw(&ret, sizeof(ret));
-        return ret;
-    }
-
-    std::string read_string(std::uint32_t len) {
-        std::vector<char> chars(len);
-        read_raw(chars.data(), len);
-        return std::string(chars.data(), len);
-    }
-
-    void write_raw(const void * ptr, size_t len) const {
-        if (len == 0) {
-            return;
-        }
-        errno = 0;
-        size_t ret = std::fwrite(ptr, len, 1, fp);
-        if (ret != 1) {
-            throw std::runtime_error(format("write error: %s", strerror(errno)));
-        }
-    }
-
-    void write_u32(std::uint32_t val) {
-        write_raw(&val, sizeof(val));
-    }
-
-    ~llama_file() {
-        if (fp) {
-            std::fclose(fp);
-        }
-    }
-};
-
-// llama_context_data
-struct llama_data_context {
-    virtual void write(const void * src, size_t size) = 0;
-    virtual size_t get_size_written() = 0;
-    virtual ~llama_data_context() = default;
-};
-
-struct llama_data_buffer_context : llama_data_context {
-    uint8_t* ptr;
-    size_t size_written = 0;
-
-    llama_data_buffer_context(uint8_t * p) : ptr(p) {}
-
-    void write(const void * src, size_t size) override {
-        memcpy(ptr, src, size);
-        ptr += size;
-        size_written += size;
-    }
-
-    size_t get_size_written() override {
-        return size_written;
-    }
-};
-
-struct llama_data_file_context : llama_data_context {
-    llama_file* file;
-    size_t size_written = 0;
-
-    llama_data_file_context(llama_file * f) : file(f) {}
-
-    void write(const void * src, size_t size) override {
-        file->write_raw(src, size);
-        size_written += size;
-    }
-
-    size_t get_size_written() override {
-        return size_written;
-    }
-};
-
-#if defined(_WIN32)
-static std::string llama_format_win_err(DWORD err) {
-    LPSTR buf;
-    size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
-                                 NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, NULL);
-    if (!size) {
-        return "FormatMessageA failed";
-    }
-    std::string ret(buf, size);
-    LocalFree(buf);
-    return ret;
-}
-#endif
-
-struct llama_mmap {
-    void * addr;
-    size_t size;
-
-    llama_mmap(const llama_mmap &) = delete;
-
-#ifdef _POSIX_MAPPED_FILES
-    static constexpr bool SUPPORTED = true;
-
-    llama_mmap(struct llama_file * file, size_t prefetch = (size_t) -1 /* -1 = max value */, bool numa = false) {
-        size = file->size;
-        int fd = fileno(file->fp);
-        int flags = MAP_SHARED;
-        // prefetch/readahead impairs performance on NUMA systems
-        if (numa) { prefetch = 0; }
-#ifdef __linux__
-        if (prefetch >= file->size) { flags |= MAP_POPULATE; }
-#endif
-        addr = mmap(NULL, file->size, PROT_READ, flags, fd, 0);
-        if (addr == MAP_FAILED) {
-            throw std::runtime_error(format("mmap failed: %s", strerror(errno)));
-        }
-
-        if (prefetch > 0) {
-            // Advise the kernel to preload the mapped memory
-            if (madvise(addr, std::min(file->size, prefetch), MADV_WILLNEED)) {
-                fprintf(stderr, "warning: madvise(.., MADV_WILLNEED) failed: %s\n",
-                        strerror(errno));
-            }
-        }
-        if (numa) {
-            // advise the kernel not to use readahead
-            // (because the next page might not belong on the same node)
-            if (madvise(addr, file->size, MADV_RANDOM)) {
-                fprintf(stderr, "warning: madvise(.., MADV_RANDOM) failed: %s\n",
-                        strerror(errno));
-            }
-        }
-    }
-
-    ~llama_mmap() {
-        munmap(addr, size);
-    }
-#elif defined(_WIN32)
-    static constexpr bool SUPPORTED = true;
-
-    llama_mmap(struct llama_file * file, bool prefetch = true, bool numa = false) {
-        (void) numa;
-
-        size = file->size;
-
-        HANDLE hFile = (HANDLE) _get_osfhandle(_fileno(file->fp));
-
-        HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL);
-        DWORD error = GetLastError();
-
-        if (hMapping == NULL) {
-            throw std::runtime_error(format("CreateFileMappingA failed: %s", llama_format_win_err(error).c_str()));
-        }
-
-        addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);
-        error = GetLastError();
-        CloseHandle(hMapping);
-
-        if (addr == NULL) {
-            throw std::runtime_error(format("MapViewOfFile failed: %s", llama_format_win_err(error).c_str()));
-        }
-
-        if (prefetch) {
-            // The PrefetchVirtualMemory API is only present on Windows 8 and above, so we
-            // will dynamically load it using GetProcAddress.
-            BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG);
-            HMODULE hKernel32;
-
-            // This call is guaranteed to succeed.
-            hKernel32 = GetModuleHandleW(L"kernel32.dll");
-
-            // This call may fail if on a pre-Win8 system.
-            pPrefetchVirtualMemory = reinterpret_cast<decltype(pPrefetchVirtualMemory)> (GetProcAddress(hKernel32, "PrefetchVirtualMemory"));
-
-            if (pPrefetchVirtualMemory) {
-                // Advise the kernel to preload the mapped memory.
-                WIN32_MEMORY_RANGE_ENTRY range;
-                range.VirtualAddress = addr;
-                range.NumberOfBytes = (SIZE_T)size;
-                if (!pPrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) {
-                    fprintf(stderr, "warning: PrefetchVirtualMemory failed: %s\n",
-                            llama_format_win_err(GetLastError()).c_str());
-                }
-            }
-        }
-    }
-
-    ~llama_mmap() {
-        if (!UnmapViewOfFile(addr)) {
-            fprintf(stderr, "warning: UnmapViewOfFile failed: %s\n",
-                    llama_format_win_err(GetLastError()).c_str());
-        }
-    }
-#else
-    static constexpr bool SUPPORTED = false;
-
-    llama_mmap(struct llama_file *, bool prefetch = true, bool numa = false) {
-        (void) prefetch;
-        (void) numa;
-
-        throw std::runtime_error(std::string("mmap not supported"));
-    }
-#endif
-};
-
-// Represents some region of memory being locked using mlock or VirtualLock;
-// will automatically unlock on destruction.
-struct llama_mlock {
-    void * addr = NULL;
-    size_t size = 0;
-    bool failed_already = false;
-
-    llama_mlock() {}
-    llama_mlock(const llama_mlock &) = delete;
-
-    ~llama_mlock() {
-        if (size) {
-            raw_unlock(addr, size);
-        }
-    }
-
-    void init(void * ptr) {
-        LLAMA_ASSERT(addr == NULL && size == 0);
-        addr = ptr;
-    }
-
-    void grow_to(size_t target_size) {
-        LLAMA_ASSERT(addr);
-        if (failed_already) {
-            return;
-        }
-        size_t granularity = lock_granularity();
-        target_size = (target_size + granularity - 1) & ~(granularity - 1);
-        if (target_size > size) {
-            if (raw_lock((uint8_t *) addr + size, target_size - size)) {
-                size = target_size;
-            } else {
-                failed_already = true;
-            }
-        }
-    }
-
-#ifdef _POSIX_MEMLOCK_RANGE
-    static constexpr bool SUPPORTED = true;
-
-    size_t lock_granularity() {
-        return (size_t) sysconf(_SC_PAGESIZE);
-    }
-
-    #ifdef __APPLE__
-        #define MLOCK_SUGGESTION \
-            "Try increasing the sysctl values 'vm.user_wire_limit' and 'vm.global_user_wire_limit' and/or " \
-            "decreasing 'vm.global_no_user_wire_amount'.  Also try increasing RLIMIT_MLOCK (ulimit -l).\n"
-    #else
-        #define MLOCK_SUGGESTION \
-            "Try increasing RLIMIT_MLOCK ('ulimit -l' as root).\n"
-    #endif
-
-    bool raw_lock(const void * addr, size_t size) {
-        if (!mlock(addr, size)) {
-            return true;
-        } else {
-            char* errmsg = std::strerror(errno);
-            bool suggest = (errno == ENOMEM);
-
-            // Check if the resource limit is fine after all
-            struct rlimit lock_limit;
-            if (suggest && getrlimit(RLIMIT_MEMLOCK, &lock_limit))
-                suggest = false;
-            if (suggest && (lock_limit.rlim_max > lock_limit.rlim_cur + size))
-                suggest = false;
-
-            fprintf(stderr, "warning: failed to mlock %zu-byte buffer (after previously locking %zu bytes): %s\n%s",
-                    size, this->size, errmsg, suggest ? MLOCK_SUGGESTION : "");
-            return false;
-        }
-    }
-
-    #undef MLOCK_SUGGESTION
-
-    void raw_unlock(void * addr, size_t size) {
-        if (munlock(addr, size)) {
-            fprintf(stderr, "warning: failed to munlock buffer: %s\n", std::strerror(errno));
-        }
-    }
-#elif defined(_WIN32)
-    static constexpr bool SUPPORTED = true;
-
-    size_t lock_granularity() {
-        SYSTEM_INFO si;
-        GetSystemInfo(&si);
-        return (size_t) si.dwPageSize;
-    }
-
-    bool raw_lock(void * ptr, size_t len) {
-        for (int tries = 1; ; tries++) {
-            if (VirtualLock(ptr, len)) {
-                return true;
-            }
-            if (tries == 2) {
-                fprintf(stderr, "warning: failed to VirtualLock %zu-byte buffer (after previously locking %zu bytes): %s\n",
-                    len, size, llama_format_win_err(GetLastError()).c_str());
-                return false;
-            }
-
-            // It failed but this was only the first try; increase the working
-            // set size and try again.
-            SIZE_T min_ws_size, max_ws_size;
-            if (!GetProcessWorkingSetSize(GetCurrentProcess(), &min_ws_size, &max_ws_size)) {
-                fprintf(stderr, "warning: GetProcessWorkingSetSize failed: %s\n",
-                        llama_format_win_err(GetLastError()).c_str());
-                return false;
-            }
-            // Per MSDN: "The maximum number of pages that a process can lock
-            // is equal to the number of pages in its minimum working set minus
-            // a small overhead."
-            // Hopefully a megabyte is enough overhead:
-            size_t increment = len + 1048576;
-            // The minimum must be <= the maximum, so we need to increase both:
-            min_ws_size += increment;
-            max_ws_size += increment;
-            if (!SetProcessWorkingSetSize(GetCurrentProcess(), min_ws_size, max_ws_size)) {
-                fprintf(stderr, "warning: SetProcessWorkingSetSize failed: %s\n",
-                        llama_format_win_err(GetLastError()).c_str());
-                return false;
-            }
-        }
-    }
-
-    void raw_unlock(void * ptr, size_t len) {
-        if (!VirtualUnlock(ptr, len)) {
-            fprintf(stderr, "warning: failed to VirtualUnlock buffer: %s\n",
-                    llama_format_win_err(GetLastError()).c_str());
-        }
-    }
-#else
-    static constexpr bool SUPPORTED = false;
-
-    size_t lock_granularity() {
-        return (size_t) 65536;
-    }
-
-    bool raw_lock(const void * addr, size_t len) {
-        fprintf(stderr, "warning: mlock not supported on this system\n");
-        return false;
-    }
-
-    void raw_unlock(const void * addr, size_t len) {}
-#endif
-};
-
-// Replacement for std::vector<uint8_t> that doesn't require zero-initialization.
-struct llama_buffer {
-    uint8_t * addr = NULL;
-    size_t size = 0;
-
-    llama_buffer() = default;
-
-    void resize(size_t len) {
-#ifdef GGML_USE_METAL
-        free(addr);
-        int result = posix_memalign((void **) &addr, getpagesize(), len);
-        if (result == 0) {
-            memset(addr, 0, len);
-        }
-        else {
-            addr = NULL;
-        }
-#else
-        delete[] addr;
-        addr = new uint8_t[len];
-#endif
-        size = len;
-    }
-
-    ~llama_buffer() {
-#ifdef GGML_USE_METAL
-        free(addr);
-#else
-        delete[] addr;
-#endif
-        addr = NULL;
-    }
-
-    // disable copy and move
-    llama_buffer(const llama_buffer&) = delete;
-    llama_buffer(llama_buffer&&) = delete;
-    llama_buffer& operator=(const llama_buffer&) = delete;
-    llama_buffer& operator=(llama_buffer&&) = delete;
-};
-
-#ifdef GGML_USE_CUBLAS
-#include "ggml-cuda.h"
-struct llama_ctx_buffer {
-    uint8_t * addr = NULL;
-    bool is_cuda;
-    size_t size = 0;
-
-    llama_ctx_buffer() = default;
-
-    void resize(size_t size) {
-        free();
-
-        addr = (uint8_t *) ggml_cuda_host_malloc(size);
-        if (addr) {
-            is_cuda = true;
-        }
-        else {
-            // fall back to pageable memory
-            addr = new uint8_t[size];
-            is_cuda = false;
-        }
-        this->size = size;
-    }
-
-    void free() {
-        if (addr) {
-            if (is_cuda) {
-                ggml_cuda_host_free(addr);
-            }
-            else {
-                delete[] addr;
-            }
-        }
-        addr = NULL;
-    }
-
-    ~llama_ctx_buffer() {
-        free();
-    }
-
-    // disable copy and move
-    llama_ctx_buffer(const llama_ctx_buffer&) = delete;
-    llama_ctx_buffer(llama_ctx_buffer&&) = delete;
-    llama_ctx_buffer& operator=(const llama_ctx_buffer&) = delete;
-    llama_ctx_buffer& operator=(llama_ctx_buffer&&) = delete;
-};
-#else
-typedef llama_buffer llama_ctx_buffer;
-#endif
-
-#endif

llm/llama.cpp/generate.go

@@ -0,0 +1,13 @@
+//go:build !darwin
+// +build !darwin
+
+package llm
+
+//go:generate git submodule init
+//go:generate git submodule update --force ggml
+//go:generate git -C ggml apply ../ggml_patch/0001-add-detokenize-endpoint.patch
+//go:generate git -C ggml apply ../ggml_patch/0002-34B-model-support.patch
+//go:generate git -C ggml apply ../ggml_patch/0003-metal-fix-synchronization-in-new-matrix-multiplicati.patch
+//go:generate git -C ggml apply ../ggml_patch/0004-metal-add-missing-barriers-for-mul-mat-2699.patch
+//go:generate cmake --fresh -S ggml -B ggml/build/cpu -DLLAMA_K_QUANTS=on
+//go:generate cmake --build ggml/build/cpu --target server --config Release

llm/llama.cpp/generate_darwin_amd64.go

@@ -0,0 +1,10 @@
+package llm
+
+//go:generate git submodule init
+//go:generate git submodule update --force ggml
+//go:generate git -C ggml apply ../ggml_patch/0001-add-detokenize-endpoint.patch
+//go:generate git -C ggml apply ../ggml_patch/0002-34B-model-support.patch
+//go:generate git -C ggml apply ../ggml_patch/0003-metal-fix-synchronization-in-new-matrix-multiplicati.patch
+//go:generate git -C ggml apply ../ggml_patch/0004-metal-add-missing-barriers-for-mul-mat-2699.patch
+//go:generate cmake --fresh -S ggml -B ggml/build/cpu -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on -DCMAKE_SYSTEM_PROCESSOR=x86_64 -DCMAKE_OSX_ARCHITECTURES=x86_64 -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0
+//go:generate cmake --build ggml/build/cpu --target server --config Release

llm/llama.cpp/generate_darwin_arm64.go

@@ -0,0 +1,10 @@
+package llm
+
+//go:generate git submodule init
+//go:generate git submodule update --force ggml
+//go:generate git -C ggml apply ../ggml_patch/0001-add-detokenize-endpoint.patch
+//go:generate git -C ggml apply ../ggml_patch/0002-34B-model-support.patch
+//go:generate git -C ggml apply ../ggml_patch/0003-metal-fix-synchronization-in-new-matrix-multiplicati.patch
+//go:generate git -C ggml apply ../ggml_patch/0004-metal-add-missing-barriers-for-mul-mat-2699.patch
+//go:generate cmake --fresh -S ggml -B ggml/build/gpu -DLLAMA_METAL=on -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on -DCMAKE_SYSTEM_PROCESSOR=arm64 -DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0
+//go:generate cmake --build ggml/build/gpu --target server --config Release

llm/llama.cpp/ggml_patch/0001-add-detokenize-endpoint.patch

@@ -0,0 +1,51 @@
+From 032ef7ff2423f5117bb59d42fb71be9cebf0a2de Mon Sep 17 00:00:00 2001
+From: Bruce MacDonald <brucewmacdonald@gmail.com>
+Date: Mon, 28 Aug 2023 18:08:12 -0400
+Subject: [PATCH] add detokenize endpoint
+
+---
+ examples/server/server.cpp | 21 +++++++++++++++++++++
+ 1 file changed, 21 insertions(+)
+
+diff --git a/examples/server/server.cpp b/examples/server/server.cpp
+index 9966045..5014691 100644
+--- a/examples/server/server.cpp
++++ b/examples/server/server.cpp
+@@ -1075,6 +1075,12 @@ static json format_tokenizer_response(const std::vector<llama_token> &tokens)
+         {"tokens", tokens}};
+ }
+ 
++static json format_detokenized_response(std::string content)
++{
++    return json{
++        {"content", content}};
++}
++
+ static void parse_options_completion(const json &body, llama_server_context &llama)
+ {
+     gpt_params default_params;
+@@ -1361,6 +1367,21 @@ int main(int argc, char **argv)
+         const json data = format_tokenizer_response(tokens);
+         return res.set_content(data.dump(), "application/json"); });
+ 
++    svr.Post("/detokenize", [&llama](const Request &req, Response &res)
++             {
++        auto lock = llama.lock();
++
++        const json body = json::parse(req.body);
++        std::string content;
++        if (body.count("tokens") != 0)
++        {
++            const std::vector<llama_token> tokens = body["tokens"];
++            content = tokens_to_str(llama.ctx, tokens.cbegin(), tokens.cend());
++        }
++
++        const json data = format_detokenized_response(content);
++        return res.set_content(data.dump(), "application/json"); });
++
+     svr.Post("/embedding", [&llama](const Request &req, Response &res)
+              {
+         auto lock = llama.lock();
+-- 
+2.39.2 (Apple Git-143)
+

llm/llama.cpp/ggml_patch/0002-34B-model-support.patch

@@ -0,0 +1,89 @@
+From 6145068a6613c37bb43a7408b5496524bdcfc402 Mon Sep 17 00:00:00 2001
+From: Bruce MacDonald <brucewmacdonald@gmail.com>
+Date: Mon, 28 Aug 2023 18:08:53 -0400
+Subject: [PATCH] 34B model support
+
+---
+ llama.cpp | 10 ++++++++++
+ 1 file changed, 10 insertions(+)
+
+diff --git a/llama.cpp b/llama.cpp
+index f2cbe76..62c5cdf 100644
+--- a/llama.cpp
++++ b/llama.cpp
+@@ -79,6 +79,7 @@ enum e_model {
+     MODEL_7B,
+     MODEL_13B,
+     MODEL_30B,
++    MODEL_34B,
+     MODEL_65B,
+     MODEL_70B,
+ };
+@@ -122,6 +123,7 @@ static std::map<e_model, size_t> MEM_REQ_SCRATCH0(int n_ctx)
+         { MODEL_7B,   ((size_t) n_ctx / 16ull + 100ull) * MB },
+         { MODEL_13B,  ((size_t) n_ctx / 12ull + 120ull) * MB },
+         { MODEL_30B,  ((size_t) n_ctx /  9ull + 160ull) * MB },
++        { MODEL_34B,  ((size_t) n_ctx / 9ull + 160ull) * MB },
+         { MODEL_65B,  ((size_t) n_ctx /  6ull + 256ull) * MB }, // guess
+         { MODEL_70B,  ((size_t) n_ctx /  7ull + 164ull) * MB },
+     };
+@@ -135,6 +137,7 @@ static const std::map<e_model, size_t> & MEM_REQ_SCRATCH1()
+         { MODEL_7B,  160ull * MB },
+         { MODEL_13B, 192ull * MB },
+         { MODEL_30B, 256ull * MB },
++        { MODEL_34B, 256ull * MB },
+         { MODEL_65B, 384ull * MB }, // guess
+         { MODEL_70B, 304ull * MB },
+     };
+@@ -149,6 +152,7 @@ static const std::map<e_model, size_t> & MEM_REQ_EVAL()
+         { MODEL_7B,  10ull * MB },
+         { MODEL_13B, 12ull * MB },
+         { MODEL_30B, 16ull * MB },
++        { MODEL_34B, 16ull * MB },
+         { MODEL_65B, 24ull * MB }, // guess
+         { MODEL_70B, 24ull * MB },
+     };
+@@ -164,6 +168,7 @@ static const std::map<e_model, size_t> & VRAM_REQ_SCRATCH_BASE()
+         { MODEL_7B,   512ull * kB },
+         { MODEL_13B,  640ull * kB },
+         { MODEL_30B,  768ull * kB },
++        { MODEL_34B,  768ull * kB },
+         { MODEL_65B, 1280ull * kB },
+         { MODEL_70B, 1280ull * kB },
+     };
+@@ -179,6 +184,7 @@ static const std::map<e_model, size_t> & VRAM_REQ_SCRATCH_PER_CONTEXT()
+         { MODEL_7B,  128ull },
+         { MODEL_13B, 160ull },
+         { MODEL_30B, 208ull },
++        { MODEL_34B, 208ull },
+         { MODEL_65B, 256ull },
+         { MODEL_70B, 256ull },
+     };
+@@ -1027,6 +1033,7 @@ static const char * llama_model_type_name(e_model type) {
+         case MODEL_7B: return "7B";
+         case MODEL_13B: return "13B";
+         case MODEL_30B: return "30B";
++        case MODEL_34B: return "34B";
+         case MODEL_65B: return "65B";
+         case MODEL_70B: return "70B";
+         default: LLAMA_ASSERT(false);
+@@ -1074,6 +1081,7 @@ static void llama_model_load_internal(
+             case 26: model.type = e_model::MODEL_3B; break;
+             case 32: model.type = e_model::MODEL_7B; break;
+             case 40: model.type = e_model::MODEL_13B; break;
++            case 48: model.type = e_model::MODEL_34B; break;
+             case 60: model.type = e_model::MODEL_30B; break;
+             case 80: model.type = e_model::MODEL_65B; break;
+             default:
+@@ -1094,6 +1102,8 @@ static void llama_model_load_internal(
+             LLAMA_LOG_WARN("%s: warning: assuming 70B model based on GQA == %d\n", __func__, n_gqa);
+             model.type = e_model::MODEL_70B;
+             hparams.f_ffn_mult = 1.3f; // from the params.json of the 70B model
++        } else if (model.type == e_model::MODEL_34B && n_gqa == 8) {
++            hparams.f_ffn_mult = 1.0f; // from the params.json of the 34B model
+         }
+ 
+         hparams.rope_freq_base  = rope_freq_base;
+-- 
+2.39.2 (Apple Git-143)
+

llm/llama.cpp/ggml_patch/0003-metal-add-missing-barriers-for-mul-mat-2699.patch

@@ -0,0 +1,32 @@
+From 8c0ea847ac1460bca534d92266e3471cb31471be Mon Sep 17 00:00:00 2001
+From: Bruce MacDonald <brucewmacdonald@gmail.com>
+Date: Tue, 5 Sep 2023 16:05:08 -0400
+Subject: [PATCH] metal: add missing barriers for mul-mat #2699
+
+---
+ ggml-metal.metal | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/ggml-metal.metal b/ggml-metal.metal
+index 3f31252..ce3541f 100644
+--- a/ggml-metal.metal
++++ b/ggml-metal.metal
+@@ -1850,6 +1850,7 @@ kernel void kernel_mul_mm(device const  uchar * src0,
+         //load data and store to threadgroup memory
+         half4x4 temp_a;
+         dequantize_func(x, il, temp_a);
++        threadgroup_barrier(mem_flags::mem_threadgroup);
+         #pragma unroll(16)
+         for (int i = 0; i < 16; i++) {
+             *(sa + SG_MAT_SIZE * ((tiitg / THREAD_PER_ROW / 8) \
+@@ -1895,6 +1896,7 @@ kernel void kernel_mul_mm(device const  uchar * src0,
+         }
+     } else {
+         // block is smaller than 64x32, we should avoid writing data outside of the matrix
++        threadgroup_barrier(mem_flags::mem_threadgroup);
+         threadgroup float *temp_str = ((threadgroup float *)shared_memory) \
+                                       + 32 * (sgitg&1) + (16 * (sgitg>>1)) * BLOCK_SIZE_M;
+         for (int i = 0; i < 8; i++) {
+-- 
+2.39.2 (Apple Git-143)
+

llm/llama.cpp/ggml_patch/0003-metal-fix-synchronization-in-new-matrix-multiplicati.patch

@@ -0,0 +1,30 @@
+From dadbed99e65252d79f81101a392d0d6497b86caa Mon Sep 17 00:00:00 2001
+From: Shouzheng Liu <lshzh.hi@gmail.com>
+Date: Mon, 21 Aug 2023 06:59:29 -0400
+Subject: [PATCH] metal : fix synchronization in new matrix multiplication
+ kernel (#2686)
+
+---
+ ggml-metal.metal | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+
+diff --git a/ggml-metal.metal b/ggml-metal.metal
+index 3f31252..88d48f6 100644
+--- a/ggml-metal.metal
++++ b/ggml-metal.metal
+@@ -1898,10 +1898,11 @@ kernel void kernel_mul_mm(device const  uchar * src0,
+         threadgroup float *temp_str = ((threadgroup float *)shared_memory) \
+                                       + 32 * (sgitg&1) + (16 * (sgitg>>1)) * BLOCK_SIZE_M;
+         for (int i = 0; i < 8; i++) {
++            threadgroup_barrier(mem_flags::mem_device);
+             simdgroup_store(c_res[i], temp_str + 8 * (i%4) + 8 * BLOCK_SIZE_M * (i/4), BLOCK_SIZE_M);
+         }
+ 
+-        threadgroup_barrier(mem_flags::mem_threadgroup);
++        threadgroup_barrier(mem_flags::mem_device);
+         device float *C = dst + BLOCK_SIZE_M * r0 + (BLOCK_SIZE_N * r1) * ne0 + im*ne1*ne0;
+         if (sgitg==0) {
+             for (int i = 0; i < n_rows; i++) {
+-- 
+2.41.0
+

llm/llama.cpp/ggml_patch/0004-metal-add-missing-barriers-for-mul-mat-2699.patch

@@ -0,0 +1,41 @@
+From 14b1d7e6f720dee41ce5a826376df738096d9033 Mon Sep 17 00:00:00 2001
+From: Shouzheng Liu <lshzh.hi@gmail.com>
+Date: Tue, 22 Aug 2023 02:18:40 -0400
+Subject: [PATCH] metal : add missing barriers for mul-mat (#2699)
+
+---
+ ggml-metal.metal | 5 +++--
+ 1 file changed, 3 insertions(+), 2 deletions(-)
+
+diff --git a/ggml-metal.metal b/ggml-metal.metal
+index 88d48f6..ce3541f 100644
+--- a/ggml-metal.metal
++++ b/ggml-metal.metal
+@@ -1850,6 +1850,7 @@ kernel void kernel_mul_mm(device const  uchar * src0,
+         //load data and store to threadgroup memory
+         half4x4 temp_a;
+         dequantize_func(x, il, temp_a);
++        threadgroup_barrier(mem_flags::mem_threadgroup);
+         #pragma unroll(16)
+         for (int i = 0; i < 16; i++) {
+             *(sa + SG_MAT_SIZE * ((tiitg / THREAD_PER_ROW / 8) \
+@@ -1895,14 +1896,14 @@ kernel void kernel_mul_mm(device const  uchar * src0,
+         }
+     } else {
+         // block is smaller than 64x32, we should avoid writing data outside of the matrix
++        threadgroup_barrier(mem_flags::mem_threadgroup);
+         threadgroup float *temp_str = ((threadgroup float *)shared_memory) \
+                                       + 32 * (sgitg&1) + (16 * (sgitg>>1)) * BLOCK_SIZE_M;
+         for (int i = 0; i < 8; i++) {
+-            threadgroup_barrier(mem_flags::mem_device);
+             simdgroup_store(c_res[i], temp_str + 8 * (i%4) + 8 * BLOCK_SIZE_M * (i/4), BLOCK_SIZE_M);
+         }
+ 
+-        threadgroup_barrier(mem_flags::mem_device);
++        threadgroup_barrier(mem_flags::mem_threadgroup);
+         device float *C = dst + BLOCK_SIZE_M * r0 + (BLOCK_SIZE_N * r1) * ne0 + im*ne1*ne0;
+         if (sgitg==0) {
+             for (int i = 0; i < n_rows; i++) {
+-- 
+2.41.0
+

llm/llama.go

@@ -1,596 +0,0 @@
-package llm
-
-/*
-#cgo CFLAGS: -Ofast -std=c11 -fPIC
-#cgo CPPFLAGS: -Ofast -Wall -Wextra -Wno-unused-function -Wno-unused-variable -DNDEBUG -DGGML_USE_K_QUANTS
-#cgo CXXFLAGS: -std=c++11 -fPIC
-#cgo darwin CPPFLAGS:  -DGGML_USE_ACCELERATE
-#cgo darwin,arm64 CPPFLAGS: -DGGML_USE_METAL -DGGML_METAL_NDEBUG
-#cgo darwin LDFLAGS: -framework Accelerate -framework Foundation -framework Metal -framework MetalKit -framework MetalPerformanceShaders
-#include <stdlib.h>
-#include "llama.h"
-
-struct llama_sample_options
-{
-	float repeat_penalty;
-	float frequency_penalty;
-	float presence_penalty;
-	float temperature;
-	int32_t top_k;
-	float top_p;
-	float tfs_z;
-	float typical_p;
-	int mirostat;
-	float mirostat_tau;
-	float mirostat_eta;
-	bool penalize_newline;
-};
-
-llama_token llama_sample(
-		struct llama_context *ctx,
-		struct llama_token_data *candidates,
-		size_t n_candidates,
-		const llama_token *last_tokens,
-		size_t n_last_tokens,
-		struct llama_sample_options *opts)
-{
-	llama_token_data_array candidates_p = {
-		candidates,
-		n_candidates,
-		false,
-	};
-
-	struct llama_token_data newline = candidates_p.data[llama_token_nl()];
-
-	llama_sample_repetition_penalty(
-		ctx, &candidates_p,
-		last_tokens, n_last_tokens,
-		opts->repeat_penalty);
-
-	llama_sample_frequency_and_presence_penalties(
-		ctx, &candidates_p,
-		last_tokens, n_last_tokens,
-		opts->frequency_penalty, opts->presence_penalty);
-
-	if (!opts->penalize_newline) {
-		candidates_p.data[llama_token_nl()] = newline;
-	}
-
-	if (opts->temperature <= 0) {
-		return llama_sample_token_greedy(ctx, &candidates_p);
-	}
-
-	if (opts->mirostat == 1) {
-		int mirostat_m = 100;
-		float mirostat_mu = 2.0f * opts->mirostat_tau;
-		llama_sample_temperature(ctx, &candidates_p, opts->temperature);
-		return llama_sample_token_mirostat(
-			ctx, &candidates_p,
-			opts->mirostat_tau, opts->mirostat_eta,
-			mirostat_m, &mirostat_mu);
-	} else if (opts->mirostat == 2) {
-		float mirostat_mu = 2.0f * opts->mirostat_tau;
-		llama_sample_temperature(ctx, &candidates_p, opts->temperature);
-		return llama_sample_token_mirostat_v2(
-			ctx, &candidates_p,
-			opts->mirostat_tau, opts->mirostat_eta,
-			&mirostat_mu);
-	} else {
-		llama_sample_top_k(ctx, &candidates_p, opts->top_k, 1);
-		llama_sample_tail_free(ctx, &candidates_p, opts->tfs_z, 1);
-		llama_sample_typical(ctx, &candidates_p, opts->typical_p, 1);
-		llama_sample_top_p(ctx, &candidates_p, opts->top_p, 1);
-		llama_sample_temperature(ctx, &candidates_p, opts->temperature);
-		return llama_sample_token(ctx, &candidates_p);
-	}
-}
-*/
-import "C"
-
-import (
-	"bytes"
-	"embed"
-	"errors"
-	"fmt"
-	"io"
-	"log"
-	"os"
-	"strings"
-	"sync"
-	"unicode/utf8"
-	"unsafe"
-
-	"github.com/jmorganca/ollama/api"
-)
-
-//go:embed ggml-metal.metal
-var fs embed.FS
-
-const ModelFamilyLlama ModelFamily = "llama"
-
-type llamaModel struct {
-	hyperparameters llamaHyperparameters
-}
-
-func (llm *llamaModel) ModelFamily() ModelFamily {
-	return ModelFamilyLlama
-}
-
-func (llm *llamaModel) ModelType() ModelType {
-	switch llm.hyperparameters.NumLayer {
-	case 26:
-		return ModelType3B
-	case 32:
-		return ModelType7B
-	case 40:
-		return ModelType13B
-	case 60:
-		return ModelType30B
-	case 80:
-		return ModelType65B
-	}
-
-	// TODO: find a better default
-	return ModelType7B
-}
-
-func (llm *llamaModel) FileType() FileType {
-	return llm.hyperparameters.FileType
-}
-
-type llamaHyperparameters struct {
-	// NumVocab is the size of the model's vocabulary.
-	NumVocab uint32
-
-	// NumEmbd is the size of the model's embedding layer.
-	NumEmbd uint32
-	NumMult uint32
-	NumHead uint32
-
-	// NumLayer is the number of layers in the model.
-	NumLayer uint32
-	NumRot   uint32
-
-	// FileType describes the quantization level of the model, e.g. Q4_0, Q5_K, etc.
-	FileType llamaFileType
-}
-
-type llamaFileType uint32
-
-const (
-	llamaFileTypeF32 llamaFileType = iota
-	llamaFileTypeF16
-	llamaFileTypeQ4_0
-	llamaFileTypeQ4_1
-	llamaFileTypeQ4_1_F16
-	llamaFileTypeQ8_0 llamaFileType = iota + 2
-	llamaFileTypeQ5_0
-	llamaFileTypeQ5_1
-	llamaFileTypeQ2_K
-	llamaFileTypeQ3_K_S
-	llamaFileTypeQ3_K_M
-	llamaFileTypeQ3_K_L
-	llamaFileTypeQ4_K_S
-	llamaFileTypeQ4_K_M
-	llamaFileTypeQ5_K_S
-	llamaFileTypeQ5_K_M
-	llamaFileTypeQ6_K
-)
-
-func (ft llamaFileType) String() string {
-	switch ft {
-	case llamaFileTypeF32:
-		return "F32"
-	case llamaFileTypeF16:
-		return "F16"
-	case llamaFileTypeQ4_0:
-		return "Q4_0"
-	case llamaFileTypeQ4_1:
-		return "Q4_1"
-	case llamaFileTypeQ4_1_F16:
-		return "Q4_1_F16"
-	case llamaFileTypeQ8_0:
-		return "Q8_0"
-	case llamaFileTypeQ5_0:
-		return "Q5_0"
-	case llamaFileTypeQ5_1:
-		return "Q5_1"
-	case llamaFileTypeQ2_K:
-		return "Q2_K"
-	case llamaFileTypeQ3_K_S:
-		return "Q3_K_S"
-	case llamaFileTypeQ3_K_M:
-		return "Q3_K_M"
-	case llamaFileTypeQ3_K_L:
-		return "Q3_K_L"
-	case llamaFileTypeQ4_K_S:
-		return "Q4_K_S"
-	case llamaFileTypeQ4_K_M:
-		return "Q4_K_M"
-	case llamaFileTypeQ5_K_S:
-		return "Q5_K_S"
-	case llamaFileTypeQ5_K_M:
-		return "Q5_K_M"
-	case llamaFileTypeQ6_K:
-		return "Q6_K"
-	default:
-		return "Unknown"
-	}
-}
-
-type llama struct {
-	params *C.struct_llama_context_params
-	model  *C.struct_llama_model
-	ctx    *C.struct_llama_context
-
-	last   []C.llama_token
-	embd   []C.llama_token
-	cursor int
-
-	mu sync.Mutex
-	gc bool
-
-	api.Options
-}
-
-func newLlama(model string, adapters []string, opts api.Options) (*llama, error) {
-	if _, err := os.Stat(model); err != nil {
-		return nil, err
-	}
-
-	llm := llama{Options: opts}
-
-	C.llama_backend_init(C.bool(llm.UseNUMA))
-
-	params := C.llama_context_default_params()
-	params.seed = C.uint(llm.Seed)
-	params.n_ctx = C.int(llm.NumCtx)
-	params.n_batch = C.int(llm.NumBatch)
-	params.n_gqa = C.int(llm.NumGQA)
-	params.n_gpu_layers = C.int(llm.NumGPU)
-	params.main_gpu = C.int(llm.MainGPU)
-	params.low_vram = C.bool(llm.LowVRAM)
-	params.f16_kv = C.bool(llm.F16KV)
-	params.logits_all = C.bool(llm.LogitsAll)
-	params.vocab_only = C.bool(llm.VocabOnly)
-	params.use_mmap = C.bool(llm.UseMMap)
-	params.use_mlock = C.bool(llm.UseMLock)
-	params.embedding = C.bool(llm.EmbeddingOnly)
-	params.rope_freq_base = C.float(llm.RopeFrequencyBase)
-	params.rope_freq_scale = C.float(llm.RopeFrequencyScale)
-
-	if len(adapters) > 0 && llm.UseMMap {
-		log.Printf("must disable mmap to use lora adapters")
-		params.use_mmap = C.bool(false)
-	}
-
-	llm.params = &params
-
-	cModel := C.CString(model)
-	defer C.free(unsafe.Pointer(cModel))
-
-	llm.model = C.llama_load_model_from_file(cModel, params)
-	if llm.model == nil {
-		return nil, errors.New("failed to load model")
-	}
-
-	llm.ctx = C.llama_new_context_with_model(llm.model, params)
-	if llm.ctx == nil {
-		return nil, errors.New("failed to create context")
-	}
-
-	for _, adapter := range adapters {
-		cAdapter := C.CString(adapter)
-		defer C.free(unsafe.Pointer(cAdapter))
-
-		if retval := C.llama_model_apply_lora_from_file(llm.model, cAdapter, nil, C.int(llm.NumThread)); retval != 0 {
-			return nil, fmt.Errorf("failed to load adapter %s", adapter)
-		}
-	}
-
-	// warm up the model
-	bos := []C.llama_token{C.llama_token_bos()}
-	C.llama_eval(llm.ctx, unsafe.SliceData(bos), C.int(len(bos)), 0, C.int(opts.NumThread))
-	C.llama_reset_timings(llm.ctx)
-
-	return &llm, nil
-}
-
-func (llm *llama) Close() {
-	llm.gc = true
-
-	llm.mu.Lock()
-	defer llm.mu.Unlock()
-
-	defer C.llama_free_model(llm.model)
-	defer C.llama_free(llm.ctx)
-
-	C.llama_print_timings(llm.ctx)
-}
-
-func (llm *llama) SetOptions(opts api.Options) {
-	llm.Options = opts
-}
-
-var errNeedMoreData = errors.New("need more data")
-
-func (llm *llama) Predict(ctx []int, prompt string, fn func(api.GenerateResponse)) error {
-	C.llama_reset_timings(llm.ctx)
-
-	llm.marshalPrompt(ctx, prompt)
-
-	C.llama_set_rng_seed(llm.ctx, C.uint(llm.Seed))
-
-	var b bytes.Buffer
-	for {
-		token, err := llm.next()
-		if llm.gc {
-			return nil
-		} else if errors.Is(err, io.EOF) {
-			break
-		} else if err != nil {
-			return err
-		}
-
-		b.WriteString(llm.Decode(int(token)))
-
-		stop, endsWithStopPrefix := handleStopSequences(&b, llm.Stop)
-		if endsWithStopPrefix {
-			continue
-		}
-
-		if utf8.Valid(b.Bytes()) || b.Len() >= utf8.UTFMax {
-			fn(api.GenerateResponse{Response: b.String()})
-			b.Reset()
-		}
-		if stop {
-			break
-		}
-	}
-
-	embd := make([]int, len(llm.embd))
-	for i := range llm.embd {
-		embd[i] = int(llm.embd[i])
-	}
-
-	timings := C.llama_get_timings(llm.ctx)
-	fn(api.GenerateResponse{
-		Done:               true,
-		Context:            embd,
-		SampleCount:        int(timings.n_sample),
-		SampleDuration:     parseDurationMs(float64(timings.t_sample_ms)),
-		PromptEvalCount:    int(timings.n_p_eval),
-		PromptEvalDuration: parseDurationMs(float64(timings.t_p_eval_ms)),
-		EvalCount:          int(timings.n_eval),
-		EvalDuration:       parseDurationMs(float64(timings.t_eval_ms)),
-	})
-
-	return nil
-}
-
-// handleStopSequences checks whether b contains any of the stop sequences, or ends with a prefix of
-// any stop sequence (and therefore might contain data that should not ultimately be returned to the
-// client).
-//
-// If b contains a stop sequence, it modifies b to remove the stop sequence and all subsequent data.
-func handleStopSequences(b *bytes.Buffer, stopSequences []string) (stop bool, endsWithStopPrefix bool) {
-	s := b.String()
-	for _, seq := range stopSequences {
-		// Check for an exact or substring match.
-		if i := strings.Index(s, seq); i != -1 {
-			b.Truncate(i)
-			return true, false
-		}
-
-		// Check if b ends with a prefix of the stop sequence.
-		if len(seq) > 1 {
-			for i := 1; i < len(seq); i++ {
-				if strings.HasSuffix(s, seq[:i]) {
-					return false, true
-				}
-			}
-		}
-	}
-
-	return false, false
-}
-
-func (llm *llama) marshalPrompt(ctx []int, prompt string) []C.llama_token {
-	tokens := append(ctx, llm.Encode(prompt)...)
-	if llm.NumKeep < 0 {
-		llm.NumKeep = len(tokens)
-	}
-
-	cTokens := make([]C.llama_token, len(tokens))
-	for i := range tokens {
-		cTokens[i] = C.llama_token(tokens[i])
-	}
-
-	// min(llm.NumCtx - 4, llm.NumKeep)
-	if llm.NumCtx-4 < llm.NumKeep {
-		llm.NumKeep = llm.NumCtx - 4
-	}
-
-	if len(tokens) >= llm.NumCtx {
-		// truncate input
-		numLeft := (llm.NumCtx - llm.NumKeep) / 2
-		truncated := cTokens[:llm.NumKeep]
-		erasedBlocks := (len(cTokens) - llm.NumKeep - numLeft - 1) / numLeft
-		truncated = append(truncated, cTokens[llm.NumKeep+erasedBlocks*numLeft:]...)
-		copy(llm.last, cTokens[len(cTokens)-llm.NumCtx:])
-
-		cTokens = truncated
-		log.Printf("input truncated: num_ctx=%d num_keep=%d num_left=%d num_tokens=%d", llm.NumCtx, llm.NumKeep, numLeft, len(truncated))
-	} else {
-		llm.last = make([]C.llama_token, llm.NumCtx-len(cTokens))
-		llm.last = append(llm.last, cTokens...)
-	}
-
-	var i int
-	for i = 0; i < len(llm.embd) && i < len(cTokens) && llm.embd[i] == cTokens[i]; i++ {
-		// noop
-	}
-
-	llm.embd = cTokens
-	if i == len(cTokens) {
-		// evaluate at least one token to generate logits
-		i--
-	}
-
-	llm.cursor = i
-
-	log.Printf("prompt: num_past=%d cached=%v eval=%v", i, len(llm.embd[:i]), len(llm.embd[i:]))
-	return cTokens
-}
-
-func (llm *llama) Encode(prompt string) []int {
-	cPrompt := C.CString(prompt)
-	defer C.free(unsafe.Pointer(cPrompt))
-
-	cTokens := make([]C.llama_token, len(prompt)+1)
-	if n := C.llama_tokenize(llm.ctx, cPrompt, unsafe.SliceData(cTokens), C.int(len(cTokens)), true); n > 0 {
-		tokens := make([]int, n)
-		for i := range cTokens[:n] {
-			tokens[i] = int(cTokens[i])
-		}
-
-		return tokens
-	}
-
-	return nil
-}
-
-func (llm *llama) Decode(tokens ...int) string {
-	var sb strings.Builder
-	for _, token := range tokens {
-		sb.WriteString(C.GoString(C.llama_token_to_str(llm.ctx, C.llama_token(token))))
-	}
-
-	return sb.String()
-}
-
-func (llm *llama) next() (C.llama_token, error) {
-	llm.mu.Lock()
-	defer llm.mu.Unlock()
-
-	if len(llm.embd) >= llm.NumCtx {
-		numLeft := (llm.NumCtx - llm.NumKeep) / 2
-		truncated := llm.embd[:llm.NumKeep]
-		truncated = append(truncated, llm.embd[len(llm.embd)-numLeft:]...)
-
-		llm.embd = truncated
-		llm.cursor = llm.NumKeep
-		log.Printf("input truncated: num_ctx=%d num_keep=%d num_left=%d num_tokens=%d cursor=%d", llm.NumCtx, llm.NumKeep, numLeft, len(truncated), llm.cursor)
-	}
-
-	for {
-		if llm.gc {
-			return 0, io.EOF
-		}
-
-		if llm.cursor >= len(llm.embd) {
-			break
-		}
-
-		numEval := len(llm.embd) - llm.cursor
-		if numEval > llm.NumBatch {
-			numEval = llm.NumBatch
-		}
-
-		if retval := C.llama_eval(llm.ctx, unsafe.SliceData(llm.embd[llm.cursor:]), C.int(numEval), C.int(llm.cursor), C.int(llm.NumThread)); retval != 0 {
-			return 0, fmt.Errorf("llama_eval: %d", retval)
-		}
-
-		llm.cursor += numEval
-	}
-
-	var sampleOpts C.struct_llama_sample_options
-	sampleOpts.repeat_penalty = C.float(llm.RepeatPenalty)
-	sampleOpts.frequency_penalty = C.float(llm.FrequencyPenalty)
-	sampleOpts.presence_penalty = C.float(llm.PresencePenalty)
-	sampleOpts.temperature = C.float(llm.Temperature)
-	sampleOpts.top_k = C.int(llm.TopK)
-	sampleOpts.top_p = C.float(llm.TopP)
-	sampleOpts.tfs_z = C.float(llm.TFSZ)
-	sampleOpts.typical_p = C.float(llm.TypicalP)
-	sampleOpts.mirostat = C.int(llm.Mirostat)
-	sampleOpts.mirostat_tau = C.float(llm.MirostatTau)
-	sampleOpts.mirostat_eta = C.float(llm.MirostatEta)
-	sampleOpts.penalize_newline = C.bool(llm.PenalizeNewline)
-
-	numVocab := C.llama_n_vocab(llm.ctx)
-	logits := unsafe.Slice(C.llama_get_logits(llm.ctx), numVocab)
-
-	// TODO: logit bias
-
-	candidates := make([]C.llama_token_data, numVocab)
-	for i := range logits {
-		candidates[i] = C.llama_token_data{
-			id:    C.int(i),
-			logit: logits[i],
-			p:     0,
-		}
-	}
-
-	repeatLastN := llm.RepeatLastN
-	if len(llm.last) < repeatLastN {
-		repeatLastN = len(llm.last)
-	}
-
-	if llm.NumCtx < repeatLastN {
-		repeatLastN = llm.NumCtx
-	}
-
-	lastN := llm.last[len(llm.last)-repeatLastN:]
-
-	token := C.llama_sample(
-		llm.ctx,
-		unsafe.SliceData(candidates), C.size_t(len(candidates)),
-		unsafe.SliceData(lastN), C.size_t(len(lastN)),
-		&sampleOpts,
-	)
-
-	llm.last = append(llm.last, token)
-	llm.embd = append(llm.embd, token)
-
-	if token == C.llama_token_eos() {
-		return 0, io.EOF
-	}
-
-	return token, nil
-}
-
-func (llm *llama) Embedding(input string) ([]float64, error) {
-	if !llm.EmbeddingOnly {
-		return nil, errors.New("llama: embedding not enabled")
-	}
-
-	tokens := llm.Encode(input)
-	if tokens == nil {
-		return nil, errors.New("llama: tokenize embedding")
-	}
-
-	cTokens := make([]C.llama_token, len(tokens))
-	for i := range tokens {
-		cTokens[i] = C.llama_token(tokens[i])
-	}
-
-	retval := C.llama_eval(llm.ctx, unsafe.SliceData(cTokens), C.int(len(tokens)), 0, C.int(llm.NumThread))
-	if retval != 0 {
-		return nil, errors.New("llama: eval")
-	}
-
-	C.llama_print_timings(llm.ctx)
-
-	n := C.llama_n_embd(llm.ctx)
-	if n <= 0 {
-		return nil, errors.New("llama: no embeddings generated")
-	}
-	cEmbeddings := unsafe.Slice(C.llama_get_embeddings(llm.ctx), n)
-
-	embeddings := make([]float64, len(cEmbeddings))
-	for i, v := range cEmbeddings {
-		embeddings[i] = float64(v)
-	}
-	return embeddings, nil
-}

llm/llama.h

@@ -1,512 +0,0 @@
-/**
- * llama.cpp - git 3ebb00935f3f0522b75df49c2769ab1774b91380
- *
- * MIT License
- *
- * Copyright (c) 2023 Georgi Gerganov
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef LLAMA_H
-#define LLAMA_H
-
-#include "ggml.h"
-#ifdef GGML_USE_CUBLAS
-#include "ggml-cuda.h"
-#define LLAMA_MAX_DEVICES GGML_CUDA_MAX_DEVICES
-#else
-#define LLAMA_MAX_DEVICES 1
-#endif // GGML_USE_CUBLAS
-#include <stddef.h>
-#include <stdint.h>
-#include <stdbool.h>
-
-#ifdef LLAMA_SHARED
-#    if defined(_WIN32) && !defined(__MINGW32__)
-#        ifdef LLAMA_BUILD
-#            define LLAMA_API __declspec(dllexport)
-#        else
-#            define LLAMA_API __declspec(dllimport)
-#        endif
-#    else
-#        define LLAMA_API __attribute__ ((visibility ("default")))
-#    endif
-#else
-#    define LLAMA_API
-#endif
-
-#ifdef __GNUC__
-#    define DEPRECATED(func, hint) func __attribute__((deprecated(hint)))
-#elif defined(_MSC_VER)
-#    define DEPRECATED(func, hint) __declspec(deprecated(hint)) func
-#else
-#    define DEPRECATED(func, hint) func
-#endif
-
-#define LLAMA_FILE_MAGIC_GGJT        0x67676a74u // 'ggjt'
-#define LLAMA_FILE_MAGIC_GGLA        0x67676c61u // 'ggla'
-#define LLAMA_FILE_MAGIC_GGMF        0x67676d66u // 'ggmf'
-#define LLAMA_FILE_MAGIC_GGML        0x67676d6cu // 'ggml'
-#define LLAMA_FILE_MAGIC_GGSN        0x6767736eu // 'ggsn'
-
-#define LLAMA_FILE_VERSION           3
-#define LLAMA_FILE_MAGIC             LLAMA_FILE_MAGIC_GGJT
-#define LLAMA_FILE_MAGIC_UNVERSIONED LLAMA_FILE_MAGIC_GGML
-#define LLAMA_SESSION_MAGIC          LLAMA_FILE_MAGIC_GGSN
-#define LLAMA_SESSION_VERSION        1
-
-#define LLAMA_DEFAULT_SEED           0xFFFFFFFF
-
-#if defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST) || defined(GGML_USE_METAL)
-// Defined when llama.cpp is compiled with support for offloading model layers to GPU.
-#define LLAMA_SUPPORTS_GPU_OFFLOAD
-#endif
-
-#ifndef LLAMA_DEFAULT_RMS_EPS
-#define LLAMA_DEFAULT_RMS_EPS 5e-6f
-#endif
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-    //
-    // C interface
-    //
-    // TODO: show sample usage
-    //
-
-    struct llama_model;
-    struct llama_context;
-
-    typedef int llama_token;
-
-    typedef struct llama_token_data {
-        llama_token id; // token id
-        float logit;    // log-odds of the token
-        float p;        // probability of the token
-    } llama_token_data;
-
-    typedef struct llama_token_data_array {
-        llama_token_data * data;
-        size_t size;
-        bool sorted;
-    } llama_token_data_array;
-
-    typedef void (*llama_progress_callback)(float progress, void *ctx);
-
-    enum llama_log_level {
-        LLAMA_LOG_LEVEL_ERROR = 2,
-        LLAMA_LOG_LEVEL_WARN  = 3,
-        LLAMA_LOG_LEVEL_INFO  = 4
-    };
-
-    // Signature for logging events
-    // Note that text includes the new line character at the end for most events.
-    // If your logging mechanism cannot handle that, check if the last character is '\n' and strip it
-    // if it exists.
-    // It might not exist for progress report where '.' is output repeatedly.
-    typedef void (*llama_log_callback)(enum llama_log_level level, const char * text, void * user_data);
-
-    struct llama_context_params {
-        uint32_t seed;         // RNG seed, -1 for random
-        int32_t  n_ctx;        // text context
-        int32_t  n_batch;      // prompt processing batch size
-        int32_t  n_gqa;        // grouped-query attention (TEMP - will be moved to model hparams)
-        float    rms_norm_eps; // rms norm epsilon (TEMP - will be moved to model hparams)
-        int32_t  n_gpu_layers; // number of layers to store in VRAM
-        int32_t  main_gpu;     // the GPU that is used for scratch and small tensors
-
-        const float * tensor_split; // how to split layers across multiple GPUs (size: LLAMA_MAX_DEVICES)
-
-        // ref: https://github.com/ggerganov/llama.cpp/pull/2054
-        float    rope_freq_base;  // RoPE base frequency
-        float    rope_freq_scale; // RoPE frequency scaling factor
-
-        // called with a progress value between 0 and 1, pass NULL to disable
-        llama_progress_callback progress_callback;
-        // context pointer passed to the progress callback
-        void * progress_callback_user_data;
-
-        // Keep the booleans together to avoid misalignment during copy-by-value.
-        bool low_vram;   // if true, reduce VRAM usage at the cost of performance
-        bool mul_mat_q;  // if true, use experimental mul_mat_q kernels
-        bool f16_kv;     // use fp16 for KV cache
-        bool logits_all; // the llama_eval() call computes all logits, not just the last one
-        bool vocab_only; // only load the vocabulary, no weights
-        bool use_mmap;   // use mmap if possible
-        bool use_mlock;  // force system to keep model in RAM
-        bool embedding;  // embedding mode only
-    };
-    // model file types
-    enum llama_ftype {
-        LLAMA_FTYPE_ALL_F32              = 0,
-        LLAMA_FTYPE_MOSTLY_F16           = 1, // except 1d tensors
-        LLAMA_FTYPE_MOSTLY_Q4_0          = 2, // except 1d tensors
-        LLAMA_FTYPE_MOSTLY_Q4_1          = 3, // except 1d tensors
-        LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16
-        // LLAMA_FTYPE_MOSTLY_Q4_2       = 5, // support has been removed
-        // LLAMA_FTYPE_MOSTLY_Q4_3       = 6, // support has been removed
-        LLAMA_FTYPE_MOSTLY_Q8_0          = 7, // except 1d tensors
-        LLAMA_FTYPE_MOSTLY_Q5_0          = 8, // except 1d tensors
-        LLAMA_FTYPE_MOSTLY_Q5_1          = 9, // except 1d tensors
-        LLAMA_FTYPE_MOSTLY_Q2_K          = 10,// except 1d tensors
-        LLAMA_FTYPE_MOSTLY_Q3_K_S        = 11,// except 1d tensors
-        LLAMA_FTYPE_MOSTLY_Q3_K_M        = 12,// except 1d tensors
-        LLAMA_FTYPE_MOSTLY_Q3_K_L        = 13,// except 1d tensors
-        LLAMA_FTYPE_MOSTLY_Q4_K_S        = 14,// except 1d tensors
-        LLAMA_FTYPE_MOSTLY_Q4_K_M        = 15,// except 1d tensors
-        LLAMA_FTYPE_MOSTLY_Q5_K_S        = 16,// except 1d tensors
-        LLAMA_FTYPE_MOSTLY_Q5_K_M        = 17,// except 1d tensors
-        LLAMA_FTYPE_MOSTLY_Q6_K          = 18,// except 1d tensors
-    };
-
-    // model quantization parameters
-    typedef struct llama_model_quantize_params {
-        int nthread;                 // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency()
-        enum llama_ftype   ftype;    // quantize to this llama_ftype
-        bool allow_requantize;       // allow quantizing non-f32/f16 tensors
-        bool quantize_output_tensor; // quantize output.weight
-    } llama_model_quantize_params;
-
-    // grammar types
-    struct llama_grammar;
-
-    // grammar element type
-    enum llama_gretype {
-        // end of rule definition
-        LLAMA_GRETYPE_END            = 0,
-
-        // start of alternate definition for rule
-        LLAMA_GRETYPE_ALT            = 1,
-
-        // non-terminal element: reference to rule
-        LLAMA_GRETYPE_RULE_REF       = 2,
-
-        // terminal element: character (code point)
-        LLAMA_GRETYPE_CHAR           = 3,
-
-        // inverse char(s) ([^a], [^a-b] [^abc])
-        LLAMA_GRETYPE_CHAR_NOT       = 4,
-
-        // modifies a preceding LLAMA_GRETYPE_CHAR or LLAMA_GRETYPE_CHAR_ALT to
-        // be an inclusive range ([a-z])
-        LLAMA_GRETYPE_CHAR_RNG_UPPER = 5,
-
-        // modifies a preceding LLAMA_GRETYPE_CHAR or
-        // LLAMA_GRETYPE_CHAR_RNG_UPPER to add an alternate char to match ([ab], [a-zA])
-        LLAMA_GRETYPE_CHAR_ALT       = 6,
-    };
-
-    typedef struct llama_grammar_element {
-        enum llama_gretype type;
-        uint32_t           value; // Unicode code point or rule ID
-    } llama_grammar_element;
-
-    // performance timing information
-    struct llama_timings {
-        double t_start_ms;
-        double t_end_ms;
-        double t_load_ms;
-        double t_sample_ms;
-        double t_p_eval_ms;
-        double t_eval_ms;
-
-        int32_t n_sample;
-        int32_t n_p_eval;
-        int32_t n_eval;
-    };
-
-    // Set callback for all future logging events.
-    // If this is not called, or NULL is supplied, everything is output on stderr.
-    LLAMA_API void llama_log_set(llama_log_callback log_callback, void * user_data);
-
-    LLAMA_API int llama_max_devices();
-
-    LLAMA_API struct llama_context_params llama_context_default_params();
-    LLAMA_API struct llama_model_quantize_params llama_model_quantize_default_params();
-
-    LLAMA_API bool llama_mmap_supported();
-    LLAMA_API bool llama_mlock_supported();
-
-    // TODO: not great API - very likely to change
-    // Initialize the llama + ggml backend
-    // If numa is true, use NUMA optimizations
-    // Call once at the start of the program
-    LLAMA_API void llama_backend_init(bool numa);
-    // Call once at the end of the program - currently only used for MPI
-    LLAMA_API void llama_backend_free();
-
-    LLAMA_API int64_t llama_time_us();
-
-    LLAMA_API struct llama_model * llama_load_model_from_file(
-                             const char * path_model,
-            struct llama_context_params   params);
-
-    LLAMA_API void llama_free_model(struct llama_model * model);
-
-    LLAMA_API struct llama_context * llama_new_context_with_model(
-                     struct llama_model * model,
-            struct llama_context_params   params);
-
-    // Various functions for loading a ggml llama model.
-    // Allocate (almost) all memory needed for the model.
-    // Return NULL on failure
-    LLAMA_API DEPRECATED(struct llama_context * llama_init_from_file(
-                             const char * path_model,
-            struct llama_context_params   params),
-            "please use llama_load_model_from_file combined with llama_new_context_with_model instead");
-
-    // Frees all allocated memory
-    LLAMA_API void llama_free(struct llama_context * ctx);
-
-    // Returns 0 on success
-    LLAMA_API int llama_model_quantize(
-            const char * fname_inp,
-            const char * fname_out,
-            const llama_model_quantize_params * params);
-
-    // Apply a LoRA adapter to a loaded model
-    // path_base_model is the path to a higher quality model to use as a base for
-    // the layers modified by the adapter. Can be NULL to use the current loaded model.
-    // The model needs to be reloaded before applying a new adapter, otherwise the adapter
-    // will be applied on top of the previous one
-    // Returns 0 on success
-    LLAMA_API DEPRECATED(int llama_apply_lora_from_file(
-            struct llama_context * ctx,
-                      const char * path_lora,
-                      const char * path_base_model,
-                             int   n_threads),
-            "please use llama_model_apply_lora_from_file instead");
-
-    LLAMA_API int llama_model_apply_lora_from_file(
-            const struct llama_model * model,
-                      const char * path_lora,
-                      const char * path_base_model,
-                             int   n_threads);
-
-    // Returns the number of tokens in the KV cache
-    LLAMA_API int llama_get_kv_cache_token_count(const struct llama_context * ctx);
-
-    // Sets the current rng seed.
-    LLAMA_API void llama_set_rng_seed(struct llama_context * ctx, uint32_t seed);
-
-    // Returns the maximum size in bytes of the state (rng, logits, embedding
-    // and kv_cache) - will often be smaller after compacting tokens
-    LLAMA_API size_t llama_get_state_size(const struct llama_context * ctx);
-
-    // Copies the state to the specified destination address.
-    // Destination needs to have allocated enough memory.
-    // Returns the number of bytes copied
-    LLAMA_API size_t llama_copy_state_data(struct llama_context * ctx, uint8_t * dst);
-
-    // Set the state reading from the specified address
-    // Returns the number of bytes read
-    LLAMA_API size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src);
-
-    // Save/load session file
-    LLAMA_API bool llama_load_session_file(struct llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out);
-    LLAMA_API bool llama_save_session_file(struct llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count);
-
-    // Run the llama inference to obtain the logits and probabilities for the next token.
-    // tokens + n_tokens is the provided batch of new tokens to process
-    // n_past is the number of tokens to use from previous eval calls
-    // Returns 0 on success
-    LLAMA_API int llama_eval(
-            struct llama_context * ctx,
-               const llama_token * tokens,
-                             int   n_tokens,
-                             int   n_past,
-                             int   n_threads);
-
-    // Same as llama_eval, but use float matrix input directly.
-    LLAMA_API int llama_eval_embd(
-            struct llama_context * ctx,
-                     const float * embd,
-                             int   n_tokens,
-                             int   n_past,
-                             int   n_threads);
-
-    // Export a static computation graph for context of 511 and batch size of 1
-    // NOTE: since this functionality is mostly for debugging and demonstration purposes, we hardcode these
-    //       parameters here to keep things simple
-    // IMPORTANT: do not use for anything else other than debugging and testing!
-    LLAMA_API int llama_eval_export(struct llama_context * ctx, const char * fname);
-
-    // Convert the provided text into tokens.
-    // The tokens pointer must be large enough to hold the resulting tokens.
-    // Returns the number of tokens on success, no more than n_max_tokens
-    // Returns a negative number on failure - the number of tokens that would have been returned
-    // TODO: not sure if correct
-    LLAMA_API int llama_tokenize(
-            struct llama_context * ctx,
-                      const char * text,
-                     llama_token * tokens,
-                             int   n_max_tokens,
-                            bool   add_bos);
-
-    LLAMA_API int llama_tokenize_with_model(
-        const struct llama_model * model,
-                      const char * text,
-                     llama_token * tokens,
-                             int   n_max_tokens,
-                            bool   add_bos);
-
-    LLAMA_API int llama_n_vocab(const struct llama_context * ctx);
-    LLAMA_API int llama_n_ctx  (const struct llama_context * ctx);
-    LLAMA_API int llama_n_embd (const struct llama_context * ctx);
-
-    LLAMA_API int llama_n_vocab_from_model(const struct llama_model * model);
-    LLAMA_API int llama_n_ctx_from_model  (const struct llama_model * model);
-    LLAMA_API int llama_n_embd_from_model (const struct llama_model * model);
-
-    // Get the vocabulary as output parameters.
-    // Returns number of results.
-    LLAMA_API int llama_get_vocab(
-            const struct llama_context * ctx,
-                          const char * * strings,
-                                 float * scores,
-                                   int   capacity);
-
-    LLAMA_API int llama_get_vocab_from_model(
-              const struct llama_model * model,
-                          const char * * strings,
-                                 float * scores,
-                                   int   capacity);
-
-    // Token logits obtained from the last call to llama_eval()
-    // The logits for the last token are stored in the last row
-    // Can be mutated in order to change the probabilities of the next token
-    // Rows: n_tokens
-    // Cols: n_vocab
-    LLAMA_API float * llama_get_logits(struct llama_context * ctx);
-
-    // Get the embeddings for the input
-    // shape: [n_embd] (1-dimensional)
-    LLAMA_API float * llama_get_embeddings(struct llama_context * ctx);
-
-    // Token Id -> String. Uses the vocabulary in the provided context
-    LLAMA_API const char * llama_token_to_str(
-            const struct llama_context * ctx,
-                           llama_token   token);
-
-    LLAMA_API const char * llama_token_to_str_with_model(
-              const struct llama_model * model,
-                           llama_token   token);
-
-    // Special tokens
-    LLAMA_API llama_token llama_token_bos();  // beginning-of-sentence
-    LLAMA_API llama_token llama_token_eos();  // end-of-sentence
-    LLAMA_API llama_token llama_token_nl();   // next-line
-
-    // Grammar
-    //
-    LLAMA_API struct llama_grammar * llama_grammar_init(
-            const llama_grammar_element ** rules,
-                                 size_t    n_rules,
-                                 size_t    start_rule_index);
-
-    LLAMA_API void llama_grammar_free(struct llama_grammar * grammar);
-
-    // Sampling functions
-
-    /// @details Repetition penalty described in CTRL academic paper https://arxiv.org/abs/1909.05858, with negative logit fix.
-    LLAMA_API void llama_sample_repetition_penalty(struct llama_context * ctx, llama_token_data_array * candidates, const llama_token * last_tokens, size_t last_tokens_size, float penalty);
-
-    /// @details Frequency and presence penalties described in OpenAI API https://platform.openai.com/docs/api-reference/parameter-details.
-    LLAMA_API void llama_sample_frequency_and_presence_penalties(struct llama_context * ctx, llama_token_data_array * candidates, const llama_token * last_tokens, size_t last_tokens_size, float alpha_frequency, float alpha_presence);
-
-    /// @details Apply classifier-free guidance to the logits as described in academic paper "Stay on topic with Classifier-Free Guidance" https://arxiv.org/abs/2306.17806
-    /// @param candidates A vector of `llama_token_data` containing the candidate tokens, the logits must be directly extracted from the original generation context without being sorted.
-    /// @params guidance_ctx A separate context from the same model. Other than a negative prompt at the beginning, it should have all generated and user input tokens copied from the main context.
-    /// @params scale Guidance strength. 1.0f means no guidance. Higher values mean stronger guidance.
-    LLAMA_API void llama_sample_classifier_free_guidance(
-              struct llama_context * ctx,
-            llama_token_data_array * candidates,
-              struct llama_context * guidance_ctx,
-                             float   scale);
-
-    /// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits.
-    LLAMA_API void llama_sample_softmax(struct llama_context * ctx, llama_token_data_array * candidates);
-
-    /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
-    LLAMA_API void llama_sample_top_k(struct llama_context * ctx, llama_token_data_array * candidates, int k, size_t min_keep);
-
-    /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
-    LLAMA_API void llama_sample_top_p(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep);
-
-    /// @details Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/.
-    LLAMA_API void llama_sample_tail_free(struct llama_context * ctx, llama_token_data_array * candidates, float z, size_t min_keep);
-
-    /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.
-    LLAMA_API void llama_sample_typical(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep);
-    LLAMA_API void llama_sample_temperature(struct llama_context * ctx, llama_token_data_array * candidates, float temp);
-
-    /// @details Apply constraints from grammar
-    LLAMA_API void llama_sample_grammar(struct llama_context * ctx, llama_token_data_array * candidates, const struct llama_grammar * grammar);
-
-    /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.
-    /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.
-    /// @param tau  The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.
-    /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.
-    /// @param m The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm.
-    /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal.
-    LLAMA_API llama_token llama_sample_token_mirostat(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, int m, float * mu);
-
-    /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.
-    /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.
-    /// @param tau  The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.
-    /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.
-    /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal.
-    LLAMA_API llama_token llama_sample_token_mirostat_v2(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, float * mu);
-
-    /// @details Selects the token with the highest probability.
-    LLAMA_API llama_token llama_sample_token_greedy(struct llama_context * ctx, llama_token_data_array * candidates);
-
-    /// @details Randomly selects a token from the candidates based on their probabilities.
-    LLAMA_API llama_token llama_sample_token(struct llama_context * ctx, llama_token_data_array * candidates);
-
-    /// @details Accepts the sampled token into the grammar
-    LLAMA_API void llama_grammar_accept_token(struct llama_context * ctx, struct llama_grammar * grammar, llama_token token);
-
-    // Performance information
-    LLAMA_API struct llama_timings llama_get_timings(struct llama_context * ctx);
-    LLAMA_API void llama_print_timings(struct llama_context * ctx);
-    LLAMA_API void llama_reset_timings(struct llama_context * ctx);
-
-    // Print system information
-    LLAMA_API const char * llama_print_system_info(void);
-
-#ifdef __cplusplus
-}
-#endif
-
-// Internal API to be implemented by llama.cpp and used by tests/benchmarks only
-#ifdef LLAMA_API_INTERNAL
-
-#include <vector>
-#include <string>
-struct ggml_tensor;
-
-const std::vector<std::pair<std::string, struct ggml_tensor *>>& llama_internal_get_tensor_map(struct llama_context * ctx);
-
-#endif
-
-#endif // LLAMA_H

llm/llama_darwin.go

@@ -1,81 +0,0 @@
-package llm
-
-import (
-	"bytes"
-	"crypto/sha256"
-	"errors"
-	"io"
-	"log"
-	"os"
-	"path/filepath"
-)
-
-func init() {
-	if err := initBackend(); err != nil {
-		log.Printf("WARNING: GPU could not be initialized correctly: %v", err)
-		log.Printf("WARNING: falling back to CPU")
-	}
-}
-
-func initBackend() error {
-	exec, err := os.Executable()
-	if err != nil {
-		return err
-	}
-
-	exec, err = filepath.EvalSymlinks(exec)
-	if err != nil {
-		return err
-	}
-
-	metal := filepath.Join(filepath.Dir(exec), "ggml-metal.metal")
-	fi, err := os.Stat(metal)
-	if err != nil && !errors.Is(err, os.ErrNotExist) {
-		return err
-	}
-
-	if fi != nil {
-		actual, err := os.Open(metal)
-		if err != nil {
-			return err
-		}
-		defer actual.Close()
-
-		actualSum := sha256.New()
-		if _, err := io.Copy(actualSum, actual); err != nil {
-			return err
-		}
-
-		expect, err := fs.Open("ggml-metal.metal")
-		if err != nil {
-			return err
-		}
-
-		expectSum := sha256.New()
-		if _, err := io.Copy(expectSum, expect); err != nil {
-			return err
-		}
-
-		if bytes.Equal(actualSum.Sum(nil), expectSum.Sum(nil)) {
-			return nil
-		}
-	}
-
-	dst, err := os.Create(filepath.Join(filepath.Dir(exec), "ggml-metal.metal"))
-	if err != nil {
-		return err
-	}
-	defer dst.Close()
-
-	src, err := fs.Open("ggml-metal.metal")
-	if err != nil {
-		return err
-	}
-	defer src.Close()
-
-	if _, err := io.Copy(dst, src); err != nil {
-		return err
-	}
-
-	return nil
-}

llm/llama_test.go

@@ -1,79 +0,0 @@
-package llm
-
-import (
-	"bytes"
-	"testing"
-)
-
-func TestCheckStopConditions(t *testing.T) {
-	tests := map[string]struct {
-		b                      string
-		stop                   []string
-		wantB                  string
-		wantStop               bool
-		wantEndsWithStopPrefix bool
-	}{
-		"not present": {
-			b:                      "abc",
-			stop:                   []string{"x"},
-			wantStop:               false,
-			wantEndsWithStopPrefix: false,
-		},
-		"exact": {
-			b:                      "abc",
-			stop:                   []string{"abc"},
-			wantStop:               true,
-			wantEndsWithStopPrefix: false,
-		},
-		"substring": {
-			b:                      "abc",
-			stop:                   []string{"b"},
-			wantB:                  "a",
-			wantStop:               true,
-			wantEndsWithStopPrefix: false,
-		},
-		"prefix 1": {
-			b:                      "abc",
-			stop:                   []string{"abcd"},
-			wantStop:               false,
-			wantEndsWithStopPrefix: true,
-		},
-		"prefix 2": {
-			b:                      "abc",
-			stop:                   []string{"bcd"},
-			wantStop:               false,
-			wantEndsWithStopPrefix: true,
-		},
-		"prefix 3": {
-			b:                      "abc",
-			stop:                   []string{"cd"},
-			wantStop:               false,
-			wantEndsWithStopPrefix: true,
-		},
-		"no prefix": {
-			b:                      "abc",
-			stop:                   []string{"bx"},
-			wantStop:               false,
-			wantEndsWithStopPrefix: false,
-		},
-	}
-	for name, test := range tests {
-		t.Run(name, func(t *testing.T) {
-			var b bytes.Buffer
-			b.WriteString(test.b)
-			stop, endsWithStopPrefix := handleStopSequences(&b, test.stop)
-			if test.wantB != "" {
-				gotB := b.String()
-				if gotB != test.wantB {
-					t.Errorf("got b %q, want %q", gotB, test.wantB)
-				}
-			}
-			if stop != test.wantStop {
-				t.Errorf("got stop %v, want %v", stop, test.wantStop)
-			}
-			if endsWithStopPrefix != test.wantEndsWithStopPrefix {
-				t.Errorf("got endsWithStopPrefix %v, want %v", endsWithStopPrefix, test.wantEndsWithStopPrefix)
-			}
-		})
-	}
-}

llm/llm.go

@@ -1,6 +1,7 @@
 package llm
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"os"
@@ -11,12 +12,13 @@ import (
 )
 
 type LLM interface {
-	Predict([]int, string, func(api.GenerateResponse)) error
-	Embedding(string) ([]float64, error)
-	Encode(string) []int
-	Decode(...int) string
+	Predict(context.Context, []int, string, func(api.GenerateResponse)) error
+	Embedding(context.Context, string) ([]float64, error)
+	Encode(context.Context, string) ([]int, error)
+	Decode(context.Context, []int) (string, error)
 	SetOptions(api.Options)
 	Close()
+	Ping(context.Context) error
 }
 
 func New(model string, adapters []string, opts api.Options) (LLM, error) {
@@ -75,7 +77,7 @@ func New(model string, adapters []string, opts api.Options) (LLM, error) {
 
 	switch ggml.ModelFamily() {
 	case ModelFamilyLlama:
-		return newLlama(model, adapters, opts)
+		return newLlama(model, adapters, ggmlRunner(), opts)
 	default:
 		return nil, fmt.Errorf("unknown ggml type: %s", ggml.ModelFamily())
 	}

llm/update-llama-cpp.sh

@@ -1,70 +0,0 @@
-#!/bin/sh
-
-set -eu
-
-
-status() { echo >&2 ">>> $*"; }
-error() { status "ERROR $*"; }
-usage() {
-    echo "usage: $(basename $0) /path/to/repo"
-    exit 1
-}
-
-OUT=$(dirname $0)
-while getopts "hC:" OPTION; do
-    case $OPTION in
-        C) OUT=$OPTARG ;;
-        *) usage ;;
-    esac
-done
-
-shift $(( $OPTIND - 1 ))
-[ $# -eq 1 ] || usage
-
-status "updating source..."
-cp -a "$1"/*.{c,h,cpp,m,metal,cu} "$OUT"
-
-status "removing incompatible files..."
-rm -f "$OUT"/build-info.h
-
-SHA1=$(git -C $1 rev-parse @)
-
-LICENSE=$(mktemp)
-cleanup() {
-    rm -f $LICENSE
-}
-trap cleanup 0
-
-cat <<EOF | sed 's/ *$//' >$LICENSE
-/**
- * llama.cpp - git $SHA1
- *
-$(sed 's/^/ * /' <$1/LICENSE)
- */
-
-EOF
-
-for IN in $OUT/*.{c,h,cpp,m,metal,cu}; do
-    TMP=$(mktemp)
-    status "updating license $IN"
-    cat $LICENSE $IN >$TMP
-    mv $TMP $IN
-done
-
-touchup() {
-    local CONSTRAINT=$1 && shift
-
-    for IN in $*; do
-        status "touching up $IN..."
-        TMP=$(mktemp)
-        {
-            echo "//go:build $CONSTRAINT"
-            echo
-        } | cat - $IN >$TMP
-        mv $TMP $IN
-    done
-}
-
-touchup darwin $OUT/ggml-metal.*
-touchup mpi $OUT/ggml-mpi.*
-touchup opencl $OUT/ggml-opencl.*

scripts/build_darwin.sh

@@ -6,8 +6,11 @@ GO_LDFLAGS="-X github.com/jmorganca/ollama/version.Version=$VERSION"
 GO_LDFLAGS="$GO_LDFLAGS -X github.com/jmorganca/ollama/server.mode=release"
 
 # build universal binary
-CGO_ENABLED=1 GOARCH=arm64 go build -ldflags "$GO_LDFLAGS" -o dist/ollama-darwin-arm64
-CGO_ENABLED=1 GOARCH=amd64 go build -ldflags "$GO_LDFLAGS" -o dist/ollama-darwin-amd64
+GOARCH=arm64 go generate ./...
+GOARCH=arm64 go build -ldflags "$GO_LDFLAGS" -o dist/ollama-darwin-arm64
+rm -rf llm/llama.cpp/ggml/build/*/bin
+GOARCH=amd64 go generate ./...
+GOARCH=amd64 go build -ldflags "$GO_LDFLAGS" -o dist/ollama-darwin-amd64
 lipo -create -output dist/ollama dist/ollama-darwin-arm64 dist/ollama-darwin-amd64
 rm dist/ollama-darwin-amd64 dist/ollama-darwin-arm64
 codesign --deep --force --options=runtime --sign "$APPLE_IDENTITY" --timestamp dist/ollama

server/images.go

@@ -9,7 +9,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"html/template"
 	"io"
 	"log"
 	"net/http"
@@ -21,6 +20,9 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"text/template"
+
+	"golang.org/x/exp/slices"
 
 	"github.com/jmorganca/ollama/api"
 	"github.com/jmorganca/ollama/llm"
@@ -39,15 +41,18 @@ type RegistryOptions struct {
 }
 
 type Model struct {
-	Name         string `json:"name"`
-	ModelPath    string
-	AdapterPaths []string
-	Template     string
-	System       string
-	Digest       string
-	ConfigDigest string
-	Options      map[string]interface{}
-	Embeddings   []vector.Embedding
+	Name          string `json:"name"`
+	ShortName     string
+	ModelPath     string
+	OriginalModel string
+	AdapterPaths  []string
+	Template      string
+	System        string
+	License       []string
+	Digest        string
+	ConfigDigest  string
+	Options       map[string]interface{}
+	Embeddings    []vector.Embedding
 }
 
 func (m *Model) Prompt(request api.GenerateRequest, embedding string) (string, error) {
@@ -111,6 +116,7 @@ type LayerReader struct {
 type ConfigV2 struct {
 	ModelFamily llm.ModelFamily `json:"model_family"`
 	ModelType   string          `json:"model_type"`
+	ModelFormat string          `json:"model_format"`
 	FileType    string          `json:"file_type"`
 	RootFS      RootFS          `json:"rootfs"`
 
@@ -169,9 +175,11 @@ func GetModel(name string) (*Model, error) {
 
 	model := &Model{
 		Name:         mp.GetFullTagname(),
+		ShortName:    mp.GetShortTagname(),
 		Digest:       digest,
 		ConfigDigest: manifest.Config.Digest,
 		Template:     "{{ .Prompt }}",
+		License:      []string{},
 	}
 
 	for _, layer := range manifest.Layers {
@@ -183,6 +191,7 @@ func GetModel(name string) (*Model, error) {
 		switch layer.MediaType {
 		case "application/vnd.ollama.image.model":
 			model.ModelPath = filename
+			model.OriginalModel = layer.From
 		case "application/vnd.ollama.image.embed":
 			file, err := os.Open(filename)
 			if err != nil {
@@ -227,6 +236,12 @@ func GetModel(name string) (*Model, error) {
 			if err = json.NewDecoder(params).Decode(&model.Options); err != nil {
 				return nil, err
 			}
+		case "application/vnd.ollama.image.license":
+			bts, err := os.ReadFile(filename)
+			if err != nil {
+				return nil, err
+			}
+			model.License = append(model.License, string(bts))
 		}
 	}
 
@@ -235,8 +250,8 @@ func GetModel(name string) (*Model, error) {
 
 func filenameWithPath(path, f string) (string, error) {
 	// if filePath starts with ~/, replace it with the user's home directory.
-	if strings.HasPrefix(f, "~/") {
-		parts := strings.Split(f, "/")
+	if strings.HasPrefix(f, fmt.Sprintf("~%s", string(os.PathSeparator))) {
+		parts := strings.Split(f, string(os.PathSeparator))
 		home, err := os.UserHomeDir()
 		if err != nil {
 			return "", fmt.Errorf("failed to open file: %v", err)
@@ -274,6 +289,7 @@ func CreateModel(ctx context.Context, name string, path string, fn func(resp api
 
 	var layers []*LayerReader
 	params := make(map[string][]string)
+	var sourceParams map[string]any
 	embed := EmbeddingParams{fn: fn}
 	for _, c := range commands {
 		log.Printf("[%s] - %s\n", c.Name, c.Args)
@@ -320,6 +336,7 @@ func CreateModel(ctx context.Context, name string, path string, fn func(resp api
 
 					config.ModelFamily = ggml.ModelFamily()
 					config.ModelType = ggml.ModelType().String()
+					config.ModelFormat = ggml.Name()
 					config.FileType = ggml.FileType().String()
 
 					// reset the file
@@ -351,12 +368,30 @@ func CreateModel(ctx context.Context, name string, path string, fn func(resp api
 					return err
 				}
 
-				// copie the model metadata
+				// copy the model metadata
 				config.ModelFamily = source.ModelFamily
 				config.ModelType = source.ModelType
+				config.ModelFormat = source.ModelFormat
 				config.FileType = source.FileType
 
 				for _, l := range mf.Layers {
+					if l.MediaType == "application/vnd.ollama.image.params" {
+						sourceParamsBlobPath, err := GetBlobsPath(l.Digest)
+						if err != nil {
+							return err
+						}
+
+						sourceParamsBlob, err := os.Open(sourceParamsBlobPath)
+						if err != nil {
+							return err
+						}
+						defer sourceParamsBlob.Close()
+
+						if err := json.NewDecoder(sourceParamsBlob).Decode(&sourceParams); err != nil {
+							return err
+						}
+					}
+
 					newLayer, err := GetLayerWithBufferFromLayer(l)
 					if err != nil {
 						return err
@@ -374,20 +409,9 @@ func CreateModel(ctx context.Context, name string, path string, fn func(resp api
 		case "adapter":
 			fn(api.ProgressResponse{Status: fmt.Sprintf("creating model %s layer", c.Name)})
 
-			fp := c.Args
-			if strings.HasPrefix(fp, "~/") {
-				parts := strings.Split(fp, "/")
-				home, err := os.UserHomeDir()
-				if err != nil {
-					return fmt.Errorf("failed to open file: %v", err)
-				}
-
-				fp = filepath.Join(home, filepath.Join(parts[1:]...))
-			}
-
-			// If filePath is not an absolute path, make it relative to the modelfile path
-			if !filepath.IsAbs(fp) {
-				fp = filepath.Join(filepath.Dir(path), fp)
+			fp, err := filenameWithPath(path, c.Args)
+			if err != nil {
+				return err
 			}
 
 			// create a model from this specified file
@@ -438,12 +462,19 @@ func CreateModel(ctx context.Context, name string, path string, fn func(resp api
 	// Create a single layer for the parameters
 	if len(params) > 0 {
 		fn(api.ProgressResponse{Status: "creating parameter layer"})
+
 		layers = removeLayerFromLayers(layers, "application/vnd.ollama.image.params")
 		formattedParams, err := formatParams(params)
 		if err != nil {
 			return fmt.Errorf("couldn't create params json: %v", err)
 		}
 
+		for k, v := range sourceParams {
+			if _, ok := formattedParams[k]; !ok {
+				formattedParams[k] = v
+			}
+		}
+
 		bts, err := json.Marshal(formattedParams)
 		if err != nil {
 			return err
@@ -521,7 +552,7 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) {
 			model = &Model{ModelPath: e.model}
 		}
 
-		if err := load(model, e.opts, defaultSessionDuration); err != nil {
+		if err := load(context.Background(), model, e.opts, defaultSessionDuration); err != nil {
 			return nil, fmt.Errorf("load model to generate embeddings: %v", err)
 		}
 
@@ -584,7 +615,7 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) {
 						embeddings = append(embeddings, vector.Embedding{Data: d, Vector: existing[d]})
 						continue
 					}
-					embed, err := loaded.llm.Embedding(d)
+					embed, err := loaded.llm.Embedding(context.Background(), d)
 					if err != nil {
 						log.Printf("failed to generate embedding for '%s' line %d: %v", filePath, i+1, err)
 						continue
@@ -641,14 +672,9 @@ func existingFileEmbeddings(digest string) (map[string][]float64, error) {
 }
 
 func removeLayerFromLayers(layers []*LayerReader, mediaType string) []*LayerReader {
-	j := 0
-	for _, l := range layers {
-		if l.MediaType != mediaType {
-			layers[j] = l
-			j++
-		}
-	}
-	return layers[:j]
+	return slices.DeleteFunc(layers, func(layer *LayerReader) bool {
+		return layer.MediaType == mediaType
+	})
 }
 
 func SaveLayers(layers []*LayerReader, fn func(resp api.ProgressResponse), force bool) error {
@@ -859,38 +885,38 @@ func DeleteModel(name string) error {
 	if err != nil {
 		return err
 	}
-	err = filepath.Walk(fp, func(path string, info os.FileInfo, err error) error {
+
+	walkFunc := func(path string, info os.FileInfo, _ error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		dir, file := filepath.Split(path)
+		dir = strings.Trim(strings.TrimPrefix(dir, fp), string(os.PathSeparator))
+		tag := strings.Join([]string{dir, file}, ":")
+		fmp := ParseModelPath(tag)
+
+		// skip the manifest we're trying to delete
+		if mp.GetFullTagname() == fmp.GetFullTagname() {
+			return nil
+		}
+
+		// save (i.e. delete from the deleteMap) any files used in other manifests
+		manifest, _, err := GetManifest(fmp)
 		if err != nil {
-			return err
+			log.Printf("skipping file: %s", fp)
+			return nil
 		}
-		if !info.IsDir() {
-			path := path[len(fp)+1:]
-			slashIndex := strings.LastIndex(path, "/")
-			if slashIndex == -1 {
-				return nil
-			}
-			tag := path[:slashIndex] + ":" + path[slashIndex+1:]
-			fmp := ParseModelPath(tag)
 
-			// skip the manifest we're trying to delete
-			if mp.GetFullTagname() == fmp.GetFullTagname() {
-				return nil
-			}
-
-			// save (i.e. delete from the deleteMap) any files used in other manifests
-			manifest, _, err := GetManifest(fmp)
-			if err != nil {
-				log.Printf("skipping file: %s", fp)
-				return nil
-			}
-			for _, layer := range manifest.Layers {
-				delete(deleteMap, layer.Digest)
-			}
-			delete(deleteMap, manifest.Config.Digest)
+		for _, layer := range manifest.Layers {
+			delete(deleteMap, layer.Digest)
 		}
+
+		delete(deleteMap, manifest.Config.Digest)
 		return nil
-	})
-	if err != nil {
+	}
+
+	if err := filepath.Walk(fp, walkFunc); err != nil {
 		return err
 	}
 
@@ -922,6 +948,83 @@ func DeleteModel(name string) error {
 	return nil
 }
 
+func ShowModelfile(model *Model) (string, error) {
+	type modelTemplate struct {
+		*Model
+		From   string
+		Params string
+	}
+
+	var params []string
+	for k, v := range model.Options {
+		switch val := v.(type) {
+		case string:
+			params = append(params, fmt.Sprintf("PARAMETER %s %s", k, val))
+		case int:
+			params = append(params, fmt.Sprintf("PARAMETER %s %s", k, strconv.Itoa(val)))
+		case float64:
+			params = append(params, fmt.Sprintf("PARAMETER %s %s", k, strconv.FormatFloat(val, 'f', 0, 64)))
+		case bool:
+			params = append(params, fmt.Sprintf("PARAMETER %s %s", k, strconv.FormatBool(val)))
+		case []interface{}:
+			for _, nv := range val {
+				switch nval := nv.(type) {
+				case string:
+					params = append(params, fmt.Sprintf("PARAMETER %s %s", k, nval))
+				case int:
+					params = append(params, fmt.Sprintf("PARAMETER %s %s", k, strconv.Itoa(nval)))
+				case float64:
+					params = append(params, fmt.Sprintf("PARAMETER %s %s", k, strconv.FormatFloat(nval, 'f', 0, 64)))
+				case bool:
+					params = append(params, fmt.Sprintf("PARAMETER %s %s", k, strconv.FormatBool(nval)))
+				default:
+					log.Printf("unknown type: %s", reflect.TypeOf(nv).String())
+				}
+			}
+		default:
+			log.Printf("unknown type: %s", reflect.TypeOf(v).String())
+		}
+	}
+
+	mt := modelTemplate{
+		Model:  model,
+		From:   model.OriginalModel,
+		Params: strings.Join(params, "\n"),
+	}
+
+	if mt.From == "" {
+		mt.From = model.ModelPath
+	}
+
+	modelFile := `# Modelfile generated by "ollama show"
+# To build a new Modelfile based on this one, replace the FROM line with:
+# FROM {{ .ShortName }}
+
+FROM {{ .From }}
+TEMPLATE """{{ .Template }}"""
+SYSTEM """{{ .System }}"""
+{{ .Params }}
+`
+	for _, l := range mt.Model.AdapterPaths {
+		modelFile += fmt.Sprintf("ADAPTER %s\n", l)
+	}
+
+	tmpl, err := template.New("").Parse(modelFile)
+	if err != nil {
+		log.Printf("error parsing template: %q", err)
+		return "", err
+	}
+
+	var buf bytes.Buffer
+
+	if err = tmpl.Execute(&buf, mt); err != nil {
+		log.Printf("error executing template: %q", err)
+		return "", err
+	}
+
+	return buf.String(), nil
+}
+
 func PushModel(ctx context.Context, name string, regOpts *RegistryOptions, fn func(api.ProgressResponse)) error {
 	mp := ParseModelPath(name)
 	fn(api.ProgressResponse{Status: "retrieving manifest"})

server/images_test.go

@@ -0,0 +1,23 @@
+package server
+
+import (
+	"testing"
+
+	"github.com/jmorganca/ollama/api"
+)
+
+func TestModelPrompt(t *testing.T) {
+	var m Model
+	req := api.GenerateRequest{
+		Template: "a{{ .Prompt }}b",
+		Prompt:   "<h1>",
+	}
+	s, err := m.Prompt(req, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "a<h1>b"
+	if s != want {
+		t.Errorf("got %q, want %q", s, want)
+	}
+}

server/modelpath.go

@@ -46,7 +46,7 @@ func ParseModelPath(name string) ModelPath {
 		name = after
 	}
 
-	parts := strings.Split(name, "/")
+	parts := strings.Split(name, string(os.PathSeparator))
 	switch len(parts) {
 	case 3:
 		mp.Registry = parts[0]
@@ -114,7 +114,12 @@ func GetManifestPath() (string, error) {
 		return "", err
 	}
 
-	return filepath.Join(home, ".ollama", "models", "manifests"), nil
+	path := filepath.Join(home, ".ollama", "models", "manifests")
+	if err := os.MkdirAll(path, 0o755); err != nil {
+		return "", err
+	}
+
+	return path, nil
 }
 
 func GetBlobsPath(digest string) (string, error) {

server/routes.go

@@ -3,17 +3,19 @@ package server
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"log"
 	"net"
 	"net/http"
 	"os"
+	"os/signal"
 	"path/filepath"
 	"reflect"
+	"strconv"
 	"strings"
 	"sync"
+	"syscall"
 	"time"
 
 	"github.com/gin-contrib/cors"
@@ -55,7 +57,7 @@ var loaded struct {
 var defaultSessionDuration = 5 * time.Minute
 
 // load a model into memory if it is not already loaded, it is up to the caller to lock loaded.mu before calling this function
-func load(model *Model, reqOpts map[string]interface{}, sessionDuration time.Duration) error {
+func load(ctx context.Context, model *Model, reqOpts map[string]interface{}, sessionDuration time.Duration) error {
 	opts := api.DefaultOptions()
 	if err := opts.FromMap(model.Options); err != nil {
 		log.Printf("could not load model options: %v", err)
@@ -67,8 +69,20 @@ func load(model *Model, reqOpts map[string]interface{}, sessionDuration time.Dur
 		return err
 	}
 
+	// check if the loaded model is still running in a subprocess, in case something unexpected happened
+	if loaded.llm != nil {
+		if err := loaded.llm.Ping(ctx); err != nil {
+			log.Print("loaded llm process not responding, closing now")
+			// the subprocess is no longer running, so close it
+			loaded.llm.Close()
+			loaded.llm = nil
+			loaded.digest = ""
+		}
+	}
+
 	if model.Digest != loaded.digest || !reflect.DeepEqual(loaded.options, opts) {
 		if loaded.llm != nil {
+			log.Println("changing loaded model")
 			loaded.llm.Close()
 			loaded.llm = nil
 			loaded.digest = ""
@@ -100,10 +114,17 @@ func load(model *Model, reqOpts map[string]interface{}, sessionDuration time.Dur
 				return err
 			}
 
-			tokensWithSystem := llmModel.Encode(promptWithSystem)
-			tokensNoSystem := llmModel.Encode(promptNoSystem)
+			tokensWithSystem, err := llmModel.Encode(ctx, promptWithSystem)
+			if err != nil {
+				return err
+			}
 
-			opts.NumKeep = len(tokensWithSystem) - len(tokensNoSystem) + 1
+			tokensNoSystem, err := llmModel.Encode(ctx, promptNoSystem)
+			if err != nil {
+				return err
+			}
+
+			opts.NumKeep = len(tokensWithSystem) - len(tokensNoSystem)
 
 			llmModel.SetOptions(opts)
 		}
@@ -151,7 +172,7 @@ func GenerateHandler(c *gin.Context) {
 	}
 
 	sessionDuration := defaultSessionDuration // TODO: set this duration from the request if specified
-	if err := load(model, req.Options, sessionDuration); err != nil {
+	if err := load(c.Request.Context(), model, req.Options, sessionDuration); err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return
 	}
@@ -160,7 +181,7 @@ func GenerateHandler(c *gin.Context) {
 
 	embedding := ""
 	if model.Embeddings != nil && len(model.Embeddings) > 0 {
-		promptEmbed, err := loaded.llm.Embedding(req.Prompt)
+		promptEmbed, err := loaded.llm.Embedding(c.Request.Context(), req.Prompt)
 		if err != nil {
 			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 			return
@@ -196,7 +217,7 @@ func GenerateHandler(c *gin.Context) {
 			ch <- r
 		}
 
-		if err := loaded.llm.Predict(req.Context, prompt, fn); err != nil {
+		if err := loaded.llm.Predict(c.Request.Context(), req.Context, prompt, fn); err != nil {
 			ch <- gin.H{"error": err.Error()}
 		}
 	}()
@@ -219,7 +240,7 @@ func EmbeddingHandler(c *gin.Context) {
 		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
 		return
 	}
-	if err := load(model, req.Options, 5*time.Minute); err != nil {
+	if err := load(c.Request.Context(), model, req.Options, 5*time.Minute); err != nil {
 		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
 		return
 	}
@@ -229,7 +250,7 @@ func EmbeddingHandler(c *gin.Context) {
 		return
 	}
 
-	embedding, err := loaded.llm.Embedding(req.Prompt)
+	embedding, err := loaded.llm.Embedding(c.Request.Context(), req.Prompt)
 	if err != nil {
 		log.Printf("embedding generation failed: %v", err)
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to generate embedding"})
@@ -344,33 +365,90 @@ func DeleteModelHandler(c *gin.Context) {
 	}
 }
 
+func ShowModelHandler(c *gin.Context) {
+	var req api.ShowRequest
+	if err := c.ShouldBindJSON(&req); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+
+	resp, err := GetModelInfo(req.Name)
+	if err != nil {
+		if os.IsNotExist(err) {
+			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Name)})
+		} else {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		}
+		return
+	}
+
+	c.JSON(http.StatusOK, resp)
+}
+
+func GetModelInfo(name string) (*api.ShowResponse, error) {
+	model, err := GetModel(name)
+	if err != nil {
+		return nil, err
+	}
+
+	resp := &api.ShowResponse{
+		License:  strings.Join(model.License, "\n"),
+		System:   model.System,
+		Template: model.Template,
+	}
+
+	mf, err := ShowModelfile(model)
+	if err != nil {
+		return nil, err
+	}
+
+	resp.Modelfile = mf
+
+	var params []string
+	cs := 30
+	for k, v := range model.Options {
+		switch val := v.(type) {
+		case string:
+			params = append(params, fmt.Sprintf("%-*s %s", cs, k, val))
+		case int:
+			params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.Itoa(val)))
+		case float64:
+			params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatFloat(val, 'f', 0, 64)))
+		case bool:
+			params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatBool(val)))
+		case []interface{}:
+			for _, nv := range val {
+				switch nval := nv.(type) {
+				case string:
+					params = append(params, fmt.Sprintf("%-*s %s", cs, k, nval))
+				case int:
+					params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.Itoa(nval)))
+				case float64:
+					params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatFloat(nval, 'f', 0, 64)))
+				case bool:
+					params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatBool(nval)))
+				}
+			}
+		}
+	}
+	resp.Parameters = strings.Join(params, "\n")
+
+	return resp, nil
+}
+
 func ListModelsHandler(c *gin.Context) {
-	var models []api.ListResponseModel
+	var models []api.ModelResponse
 	fp, err := GetManifestPath()
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return
 	}
-	err = filepath.Walk(fp, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			if errors.Is(err, os.ErrNotExist) {
-				log.Printf("manifest file does not exist: %s", fp)
-				return nil
-			}
-			return err
-		}
+
+	walkFunc := func(path string, info os.FileInfo, _ error) error {
 		if !info.IsDir() {
-			fi, err := os.Stat(path)
-			if err != nil {
-				log.Printf("skipping file: %s", fp)
-				return nil
-			}
-			path := path[len(fp)+1:]
-			slashIndex := strings.LastIndex(path, "/")
-			if slashIndex == -1 {
-				return nil
-			}
-			tag := path[:slashIndex] + ":" + path[slashIndex+1:]
+			dir, file := filepath.Split(path)
+			dir = strings.Trim(strings.TrimPrefix(dir, fp), string(os.PathSeparator))
+			tag := strings.Join([]string{dir, file}, ":")
 
 			mp := ParseModelPath(tag)
 			manifest, digest, err := GetManifest(mp)
@@ -378,17 +456,19 @@ func ListModelsHandler(c *gin.Context) {
 				log.Printf("skipping file: %s", fp)
 				return nil
 			}
-			model := api.ListResponseModel{
+
+			models = append(models, api.ModelResponse{
 				Name:       mp.GetShortTagname(),
 				Size:       manifest.GetTotalSize(),
 				Digest:     digest,
-				ModifiedAt: fi.ModTime(),
-			}
-			models = append(models, model)
+				ModifiedAt: info.ModTime(),
+			})
 		}
+
 		return nil
-	})
-	if err != nil {
+	}
+
+	if err := filepath.Walk(fp, walkFunc); err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return
 	}
@@ -449,12 +529,24 @@ func Serve(ln net.Listener, origins []string) error {
 	r.POST("/api/copy", CopyModelHandler)
 	r.GET("/api/tags", ListModelsHandler)
 	r.DELETE("/api/delete", DeleteModelHandler)
+	r.POST("/api/show", ShowModelHandler)
 
 	log.Printf("Listening on %s", ln.Addr())
 	s := &http.Server{
 		Handler: r,
 	}
 
+	// listen for a ctrl+c and stop any loaded llm
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, syscall.SIGINT)
+	go func() {
+		<-signals
+		if loaded.llm != nil {
+			loaded.llm.Close()
+		}
+		os.Exit(0)
+	}()
+
 	return s.Serve(ln)
 }