OpenAI Delete Endpoint

Actually skip PhysX on windows

.github/workflows/release.yaml

@@ -28,6 +28,7 @@ jobs:
           security unlock-keychain -p password build.keychain
           security import certificate.p12 -k build.keychain -P $MACOS_SIGNING_KEY_PASSWORD -T /usr/bin/codesign
           security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k password build.keychain
+          security set-keychain-settings -lut 3600 build.keychain
       - uses: actions/setup-go@v5
         with:
           go-version-file: go.mod

.github/workflows/test.yaml

@@ -34,13 +34,13 @@ jobs:
             git diff-tree -r --no-commit-id --name-only \
               $(git merge-base ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }}) \
               ${{ github.event.pull_request.head.sha }} \
-              | xargs python3 -c "import sys; print(any([x.startswith('$1') for x in sys.argv[1:]]))"
+              | xargs python3 -c "import sys; from pathlib import Path; print(any(Path(x).match(glob) for x in sys.argv[1:] for glob in '$*'.split(' ')))"
           }
 
           {
-            echo GENERATE=$(changed llm/)
-            echo GENERATE_CUDA=$(changed llm/)
-            echo GENERATE_ROCM=$(changed llm/)
+            echo GENERATE=$(changed 'llm/llama.cpp' 'llm/patches/**' 'llm/ext_server/**' 'llm/generate/**')
+            echo GENERATE_CUDA=$(changed 'llm/llama.cpp' 'llm/patches/**' 'llm/ext_server/**' 'llm/generate/**')
+            echo GENERATE_ROCM=$(changed 'llm/llama.cpp' 'llm/patches/**' 'llm/ext_server/**' 'llm/generate/**')
           } >>$GITHUB_OUTPUT
 
   generate:
@@ -269,9 +269,9 @@ jobs:
           mkdir -p llm/build/darwin/$ARCH/stub/bin
           touch llm/build/darwin/$ARCH/stub/bin/ollama_llama_server
         if: ${{ startsWith(matrix.os, 'macos-') }}
-      - uses: golangci/golangci-lint-action@v4
+      - uses: golangci/golangci-lint-action@v6
         with:
-          args: --timeout 8m0s -v
+          args: --timeout 8m0s -v ${{ startsWith(matrix.os, 'windows-') && '' || '--disable gofmt --disable goimports' }}
   test:
     strategy:
       matrix:
@@ -287,6 +287,8 @@ jobs:
       GOARCH: ${{ matrix.arch }}
       CGO_ENABLED: '1'
       OLLAMA_CPU_TARGET: 'static'
+      OLLAMA_SKIP_CPU_GENERATE: '1'
+      OLLAMA_SKIP_METAL_GENERATE: '1'
     steps:
       - uses: actions/checkout@v4
         with:

.golangci.yaml

@@ -9,9 +9,26 @@ linters:
     - contextcheck
     - exportloopref
     - gocheckcompilerdirectives
-    # FIXME: for some reason this errors on windows
+    # conditionally enable this on linux/macos
     # - gofmt
     # - goimports
+    - intrange
     - misspell
     - nilerr
+    - nolintlint
+    - nosprintfhostport
+    - testifylint
+    - unconvert
     - unused
+    - wastedassign
+    - whitespace
+    - usestdlibvars
+severity:
+  default-severity: error
+  rules:
+    - linters:
+        - gofmt
+        - goimports
+        - intrange
+        - usestdlibvars
+      severity: info

README.md

@@ -6,7 +6,7 @@
 
 [![Discord](https://dcbadge.vercel.app/api/server/ollama?style=flat&compact=true)](https://discord.gg/ollama)
 
-Get up and running with large language models locally.
+Get up and running with large language models.
 
 ### macOS
 
@@ -51,15 +51,17 @@ Here are some example models that can be downloaded:
 | ------------------ | ---------- | ----- | ------------------------------ |
 | Llama 3            | 8B         | 4.7GB | `ollama run llama3`            |
 | Llama 3            | 70B        | 40GB  | `ollama run llama3:70b`        |
-| Phi-3              | 3.8B       | 2.3GB | `ollama run phi3`              |
+| Phi 3 Mini         | 3.8B       | 2.3GB | `ollama run phi3`              |
+| Phi 3 Medium       | 14B        | 7.9GB | `ollama run phi3:medium`       |
+| Gemma              | 2B         | 1.4GB | `ollama run gemma:2b`          |
+| Gemma              | 7B         | 4.8GB | `ollama run gemma:7b`          |
 | Mistral            | 7B         | 4.1GB | `ollama run mistral`           |
+| Moondream 2        | 1.4B       | 829MB | `ollama run moondream`         |
 | Neural Chat        | 7B         | 4.1GB | `ollama run neural-chat`       |
 | Starling           | 7B         | 4.1GB | `ollama run starling-lm`       |
 | Code Llama         | 7B         | 3.8GB | `ollama run codellama`         |
 | Llama 2 Uncensored | 7B         | 3.8GB | `ollama run llama2-uncensored` |
 | LLaVA              | 7B         | 4.5GB | `ollama run llava`             |
-| Gemma              | 2B         | 1.4GB | `ollama run gemma:2b`          |
-| Gemma              | 7B         | 4.8GB | `ollama run gemma:7b`          |
 | Solar              | 10.7B      | 6.1GB | `ollama run solar`             |
 
 > Note: You should have at least 8 GB of RAM available to run the 7B models, 16 GB to run the 13B models, and 32 GB to run the 33B models.
@@ -192,25 +194,7 @@ ollama list
 
 ## Building
 
-Install `cmake` and `go`:
-
-```
-brew install cmake go
-```
-
-Then generate dependencies:
-
-```
-go generate ./...
-```
-
-Then build the binary:
-
-```
-go build .
-```
-
-More detailed instructions can be found in the [developer guide](https://github.com/ollama/ollama/blob/main/docs/development.md)
+See the [developer guide](https://github.com/ollama/ollama/blob/main/docs/development.md)
 
 ### Running local builds
 
@@ -299,6 +283,9 @@ See the [API documentation](./docs/api.md) for all endpoints.
 - [Ollama RAG Chatbot](https://github.com/datvodinh/rag-chatbot.git) (Local Chat with multiple PDFs using Ollama and RAG)
 - [BrainSoup](https://www.nurgo-software.com/products/brainsoup) (Flexible native client with RAG & multi-agent automation)
 - [macai](https://github.com/Renset/macai) (macOS client for Ollama, ChatGPT, and other compatible API back-ends)
+- [Olpaka](https://github.com/Otacon/olpaka) (User-friendly Flutter Web App for Ollama)
+- [OllamaSpring](https://github.com/CrazyNeil/OllamaSpring) (Ollama Client for macOS)
+- [LLocal.in](https://github.com/kartikm7/llocal) (Easy to use Electron Desktop Client for Ollama)
 
 ### Terminal
 
@@ -321,6 +308,7 @@ See the [API documentation](./docs/api.md) for all endpoints.
 - [ShellOracle](https://github.com/djcopley/ShellOracle)
 - [tlm](https://github.com/yusufcanb/tlm)
 - [podman-ollama](https://github.com/ericcurtin/podman-ollama)
+- [gollama](https://github.com/sammcj/gollama)
 
 ### Database
 
@@ -338,11 +326,13 @@ See the [API documentation](./docs/api.md) for all endpoints.
 - [LangChain](https://python.langchain.com/docs/integrations/llms/ollama) and [LangChain.js](https://js.langchain.com/docs/modules/model_io/models/llms/integrations/ollama) with [example](https://js.langchain.com/docs/use_cases/question_answering/local_retrieval_qa)
 - [LangChainGo](https://github.com/tmc/langchaingo/) with [example](https://github.com/tmc/langchaingo/tree/main/examples/ollama-completion-example)
 - [LangChain4j](https://github.com/langchain4j/langchain4j) with [example](https://github.com/langchain4j/langchain4j-examples/tree/main/ollama-examples/src/main/java)
+- [LangChainRust](https://github.com/Abraxas-365/langchain-rust) with [example](https://github.com/Abraxas-365/langchain-rust/blob/main/examples/llm_ollama.rs)
 - [LlamaIndex](https://gpt-index.readthedocs.io/en/stable/examples/llm/ollama.html)
 - [LiteLLM](https://github.com/BerriAI/litellm)
 - [OllamaSharp for .NET](https://github.com/awaescher/OllamaSharp)
 - [Ollama for Ruby](https://github.com/gbaptista/ollama-ai)
 - [Ollama-rs for Rust](https://github.com/pepperoni21/ollama-rs)
+- [Ollama-hpp for C++](https://github.com/jmont-dev/ollama-hpp)
 - [Ollama4j for Java](https://github.com/amithkoujalgi/ollama4j)
 - [ModelFusion Typescript Library](https://modelfusion.dev/integration/model-provider/ollama)
 - [OllamaKit for Swift](https://github.com/kevinhermawan/OllamaKit)
@@ -359,7 +349,8 @@ See the [API documentation](./docs/api.md) for all endpoints.
 - [Testcontainers](https://testcontainers.com/modules/ollama/)
 - [Portkey](https://portkey.ai/docs/welcome/integration-guides/ollama)
 - [PromptingTools.jl](https://github.com/svilupp/PromptingTools.jl) with an [example](https://svilupp.github.io/PromptingTools.jl/dev/examples/working_with_ollama)
-- [LlamaScript](https://github.com/WolfTheDeveloper/llamascript)
+- [LlamaScript](https://github.com/Project-Llama/llamascript)
+
 ### Mobile
 
 - [Enchanted](https://github.com/AugustDev/enchanted)
@@ -391,7 +382,10 @@ See the [API documentation](./docs/api.md) for all endpoints.
 - [AI Telegram Bot](https://github.com/tusharhero/aitelegrambot) (Telegram bot using Ollama in backend)
 - [AI ST Completion](https://github.com/yaroslavyaroslav/OpenAI-sublime-text) (Sublime Text 4 AI assistant plugin with Ollama support)
 - [Discord-Ollama Chat Bot](https://github.com/kevinthedang/discord-ollama) (Generalized TypeScript Discord Bot w/ Tuning Documentation)
+- [Discord AI chat/moderation bot](https://github.com/rapmd73/Companion) Chat/moderation bot written in python. Uses Ollama to create personalities.
+- [Headless Ollama](https://github.com/nischalj10/headless-ollama) (Scripts to automatically install ollama client & models on any OS for apps that depends on ollama server)
+
+### Supported backends
 
-### Supported backends 
 - [llama.cpp](https://github.com/ggerganov/llama.cpp) project founded by Georgi Gerganov.
 

api/client.go

@@ -23,11 +23,9 @@ import (
 	"net"
 	"net/http"
 	"net/url"
-	"os"
 	"runtime"
-	"strconv"
-	"strings"
 
+	"github.com/ollama/ollama/envconfig"
 	"github.com/ollama/ollama/format"
 	"github.com/ollama/ollama/version"
 )
@@ -65,10 +63,7 @@ func checkError(resp *http.Response, body []byte) error {
 // If the variable is not specified, a default ollama host and port will be
 // used.
 func ClientFromEnvironment() (*Client, error) {
-	ollamaHost, err := GetOllamaHost()
-	if err != nil {
-		return nil, err
-	}
+	ollamaHost := envconfig.Host
 
 	return &Client{
 		base: &url.URL{
@@ -79,52 +74,6 @@ func ClientFromEnvironment() (*Client, error) {
 	}, nil
 }
 
-type OllamaHost struct {
-	Scheme string
-	Host   string
-	Port   string
-}
-
-func GetOllamaHost() (OllamaHost, error) {
-	defaultPort := "11434"
-
-	hostVar := os.Getenv("OLLAMA_HOST")
-	hostVar = strings.TrimSpace(strings.Trim(strings.TrimSpace(hostVar), "\"'"))
-
-	scheme, hostport, ok := strings.Cut(hostVar, "://")
-	switch {
-	case !ok:
-		scheme, hostport = "http", hostVar
-	case scheme == "http":
-		defaultPort = "80"
-	case scheme == "https":
-		defaultPort = "443"
-	}
-
-	// trim trailing slashes
-	hostport = strings.TrimRight(hostport, "/")
-
-	host, port, err := net.SplitHostPort(hostport)
-	if err != nil {
-		host, port = "127.0.0.1", defaultPort
-		if ip := net.ParseIP(strings.Trim(hostport, "[]")); ip != nil {
-			host = ip.String()
-		} else if hostport != "" {
-			host = hostport
-		}
-	}
-
-	if portNum, err := strconv.ParseInt(port, 10, 32); err != nil || portNum > 65535 || portNum < 0 {
-		return OllamaHost{}, ErrInvalidHostPort
-	}
-
-	return OllamaHost{
-		Scheme: scheme,
-		Host:   host,
-		Port:   port,
-	}, nil
-}
-
 func NewClient(base *url.URL, http *http.Client) *Client {
 	return &Client{
 		base: base,
@@ -354,6 +303,15 @@ func (c *Client) List(ctx context.Context) (*ListResponse, error) {
 	return &lr, nil
 }
 
+// List running models.
+func (c *Client) ListRunning(ctx context.Context) (*ProcessResponse, error) {
+	var lr ProcessResponse
+	if err := c.do(ctx, http.MethodGet, "/api/ps", nil, &lr); err != nil {
+		return nil, err
+	}
+	return &lr, nil
+}
+
 // Copy copies a model - creating a model with another name from an existing
 // model.
 func (c *Client) Copy(ctx context.Context, req *CopyRequest) error {

api/client_test.go

@@ -1,11 +1,9 @@
 package api
 
 import (
-	"fmt"
-	"net"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
+	"github.com/ollama/ollama/envconfig"
 )
 
 func TestClientFromEnvironment(t *testing.T) {
@@ -35,6 +33,7 @@ func TestClientFromEnvironment(t *testing.T) {
 	for k, v := range testCases {
 		t.Run(k, func(t *testing.T) {
 			t.Setenv("OLLAMA_HOST", v.value)
+			envconfig.LoadConfig()
 
 			client, err := ClientFromEnvironment()
 			if err != v.err {
@@ -46,40 +45,4 @@ func TestClientFromEnvironment(t *testing.T) {
 			}
 		})
 	}
-
-	hostTestCases := map[string]*testCase{
-		"empty":               {value: "", expect: "127.0.0.1:11434"},
-		"only address":        {value: "1.2.3.4", expect: "1.2.3.4:11434"},
-		"only port":           {value: ":1234", expect: ":1234"},
-		"address and port":    {value: "1.2.3.4:1234", expect: "1.2.3.4:1234"},
-		"hostname":            {value: "example.com", expect: "example.com:11434"},
-		"hostname and port":   {value: "example.com:1234", expect: "example.com:1234"},
-		"zero port":           {value: ":0", expect: ":0"},
-		"too large port":      {value: ":66000", err: ErrInvalidHostPort},
-		"too small port":      {value: ":-1", err: ErrInvalidHostPort},
-		"ipv6 localhost":      {value: "[::1]", expect: "[::1]:11434"},
-		"ipv6 world open":     {value: "[::]", expect: "[::]:11434"},
-		"ipv6 no brackets":    {value: "::1", expect: "[::1]:11434"},
-		"ipv6 + port":         {value: "[::1]:1337", expect: "[::1]:1337"},
-		"extra space":         {value: " 1.2.3.4 ", expect: "1.2.3.4:11434"},
-		"extra quotes":        {value: "\"1.2.3.4\"", expect: "1.2.3.4:11434"},
-		"extra space+quotes":  {value: " \" 1.2.3.4 \" ", expect: "1.2.3.4:11434"},
-		"extra single quotes": {value: "'1.2.3.4'", expect: "1.2.3.4:11434"},
-	}
-
-	for k, v := range hostTestCases {
-		t.Run(k, func(t *testing.T) {
-			t.Setenv("OLLAMA_HOST", v.value)
-
-			oh, err := GetOllamaHost()
-			if err != v.err {
-				t.Fatalf("expected %s, got %s", v.err, err)
-			}
-
-			if err == nil {
-				host := net.JoinHostPort(oh.Host, oh.Port)
-				assert.Equal(t, v.expect, host, fmt.Sprintf("%s: expected %s, got %s", k, v.expect, host))
-			}
-		})
-	}
 }

api/types.go

@@ -2,7 +2,6 @@ package api
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
 	"log/slog"
 	"math"
@@ -282,11 +281,16 @@ type PushRequest struct {
 
 // ListResponse is the response from [Client.List].
 type ListResponse struct {
-	Models []ModelResponse `json:"models"`
+	Models []ListModelResponse `json:"models"`
 }
 
-// ModelResponse is a single model description in [ListResponse].
-type ModelResponse struct {
+// ProcessResponse is the response from [Client.Process].
+type ProcessResponse struct {
+	Models []ProcessModelResponse `json:"models"`
+}
+
+// ListModelResponse is a single model description in [ListResponse].
+type ListModelResponse struct {
 	Name       string       `json:"name"`
 	Model      string       `json:"model"`
 	ModifiedAt time.Time    `json:"modified_at"`
@@ -295,6 +299,17 @@ type ModelResponse struct {
 	Details    ModelDetails `json:"details,omitempty"`
 }
 
+// ProcessModelResponse is a single model description in [ProcessResponse].
+type ProcessModelResponse struct {
+	Name      string       `json:"name"`
+	Model     string       `json:"model"`
+	Size      int64        `json:"size"`
+	Digest    string       `json:"digest"`
+	Details   ModelDetails `json:"details,omitempty"`
+	ExpiresAt time.Time    `json:"expires_at"`
+	SizeVRAM  int64        `json:"size_vram"`
+}
+
 type TokenResponse struct {
 	Token string `json:"token"`
 }
@@ -304,7 +319,7 @@ type GenerateResponse struct {
 	// Model is the model name that generated the response.
 	Model string `json:"model"`
 
-	//CreatedAt is the timestamp of the response.
+	// CreatedAt is the timestamp of the response.
 	CreatedAt time.Time `json:"created_at"`
 
 	// Response is the textual response itself.
@@ -361,8 +376,6 @@ func (m *Metrics) Summary() {
 	}
 }
 
-var ErrInvalidHostPort = errors.New("invalid port specified in OLLAMA_HOST")
-
 func (opts *Options) FromMap(m map[string]interface{}) error {
 	valueOpts := reflect.ValueOf(opts).Elem() // names of the fields in the options struct
 	typeOpts := reflect.TypeOf(opts).Elem()   // types of the fields in the options struct

api/types_test.go

@@ -72,13 +72,13 @@ func TestDurationMarshalUnmarshal(t *testing.T) {
 		},
 		{
 			"positive duration",
-			time.Duration(42 * time.Second),
-			time.Duration(42 * time.Second),
+			42 * time.Second,
+			42 * time.Second,
 		},
 		{
 			"another positive duration",
-			time.Duration(42 * time.Minute),
-			time.Duration(42 * time.Minute),
+			42 * time.Minute,
+			42 * time.Minute,
 		},
 		{
 			"zero duration",

app/lifecycle/logging.go

@@ -6,7 +6,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/ollama/ollama/server/envconfig"
+	"github.com/ollama/ollama/envconfig"
 )
 
 func InitLogging() {

app/lifecycle/paths.go

@@ -69,7 +69,6 @@ func init() {
 				slog.Error(fmt.Sprintf("create ollama dir %s: %v", AppDataDir, err))
 			}
 		}
-
 	} else if runtime.GOOS == "darwin" {
 		// TODO
 		AppName += ".app"

app/lifecycle/server.go

@@ -15,7 +15,7 @@ import (
 )
 
 func getCLIFullPath(command string) string {
-	cmdPath := ""
+	var cmdPath string
 	appExe, err := os.Executable()
 	if err == nil {
 		cmdPath = filepath.Join(filepath.Dir(appExe), command)
@@ -65,7 +65,6 @@ func start(ctx context.Context, command string) (*exec.Cmd, error) {
 	if err != nil {
 		if !errors.Is(err, os.ErrNotExist) {
 			return nil, fmt.Errorf("stat ollama server log dir %s: %v", logDir, err)
-
 		}
 
 		if err := os.MkdirAll(logDir, 0o755); err != nil {

app/lifecycle/server_windows.go

@@ -24,7 +24,8 @@ func terminate(cmd *exec.Cmd) error {
 	if err != nil {
 		return err
 	}
-	defer dll.Release() // nolint: errcheck
+	//nolint:errcheck
+	defer dll.Release()
 
 	pid := cmd.Process.Pid
 
@@ -73,7 +74,8 @@ func isProcessExited(pid int) (bool, error) {
 	if err != nil {
 		return false, fmt.Errorf("failed to open process: %v", err)
 	}
-	defer windows.CloseHandle(hProcess) // nolint: errcheck
+	//nolint:errcheck
+	defer windows.CloseHandle(hProcess)
 
 	var exitCode uint32
 	err = windows.GetExitCodeProcess(hProcess, &exitCode)

app/lifecycle/updater.go

@@ -78,7 +78,7 @@ func IsNewReleaseAvailable(ctx context.Context) (bool, UpdateResponse) {
 	}
 	defer resp.Body.Close()
 
-	if resp.StatusCode == 204 {
+	if resp.StatusCode == http.StatusNoContent {
 		slog.Debug("check update response 204 (current version is up to date)")
 		return false, updateResp
 	}
@@ -87,7 +87,7 @@ func IsNewReleaseAvailable(ctx context.Context) (bool, UpdateResponse) {
 		slog.Warn(fmt.Sprintf("failed to read body response: %s", err))
 	}
 
-	if resp.StatusCode != 200 {
+	if resp.StatusCode != http.StatusOK {
 		slog.Info(fmt.Sprintf("check update error %d - %.96s", resp.StatusCode, string(body)))
 		return false, updateResp
 	}
@@ -114,7 +114,7 @@ func DownloadNewRelease(ctx context.Context, updateResp UpdateResponse) error {
 	if err != nil {
 		return fmt.Errorf("error checking update: %w", err)
 	}
-	if resp.StatusCode != 200 {
+	if resp.StatusCode != http.StatusOK {
 		return fmt.Errorf("unexpected status attempting to download update %d", resp.StatusCode)
 	}
 	resp.Body.Close()

app/ollama_welcome.ps1

@@ -4,5 +4,5 @@ write-host "Welcome to Ollama!"
 write-host ""
 write-host "Run your first model:"
 write-host ""
-write-host "`tollama run llama2"
+write-host "`tollama run llama3"
 write-host ""

app/store/store.go

@@ -29,7 +29,6 @@ func GetID() string {
 		initStore()
 	}
 	return store.ID
-
 }
 
 func GetFirstTimeRun() bool {

app/tray/wintray/eventloop.go

@@ -47,7 +47,6 @@ func nativeLoop() {
 		default:
 			pTranslateMessage.Call(uintptr(unsafe.Pointer(m))) //nolint:errcheck
 			pDispatchMessage.Call(uintptr(unsafe.Pointer(m)))  //nolint:errcheck
-
 		}
 	}
 }
@@ -160,8 +159,8 @@ func (t *winTray) wndProc(hWnd windows.Handle, message uint32, wParam, lParam ui
 		lResult, _, _ = pDefWindowProc.Call(
 			uintptr(hWnd),
 			uintptr(message),
-			uintptr(wParam),
-			uintptr(lParam),
+			wParam,
+			lParam,
 		)
 	}
 	return

app/tray/wintray/tray.go

@@ -186,7 +186,7 @@ func (t *winTray) initInstance() error {
 	t.muNID.Lock()
 	defer t.muNID.Unlock()
 	t.nid = &notifyIconData{
-		Wnd:             windows.Handle(t.window),
+		Wnd:             t.window,
 		ID:              100,
 		Flags:           NIF_MESSAGE,
 		CallbackMessage: t.wmSystrayMessage,
@@ -197,7 +197,6 @@ func (t *winTray) initInstance() error {
 }
 
 func (t *winTray) createMenu() error {
-
 	menuHandle, _, err := pCreatePopupMenu.Call()
 	if menuHandle == 0 {
 		return err
@@ -246,7 +245,7 @@ func (t *winTray) addOrUpdateMenuItem(menuItemId uint32, parentId uint32, title
 	mi := menuItemInfo{
 		Mask:     MIIM_FTYPE | MIIM_STRING | MIIM_ID | MIIM_STATE,
 		Type:     MFT_STRING,
-		ID:       uint32(menuItemId),
+		ID:       menuItemId,
 		TypeData: titlePtr,
 		Cch:      uint32(len(title)),
 	}
@@ -302,11 +301,10 @@ func (t *winTray) addOrUpdateMenuItem(menuItemId uint32, parentId uint32, title
 }
 
 func (t *winTray) addSeparatorMenuItem(menuItemId, parentId uint32) error {
-
 	mi := menuItemInfo{
 		Mask: MIIM_FTYPE | MIIM_ID | MIIM_STATE,
 		Type: MFT_SEPARATOR,
-		ID:   uint32(menuItemId),
+		ID:   menuItemId,
 	}
 
 	mi.Size = uint32(unsafe.Sizeof(mi))
@@ -426,7 +424,6 @@ func iconBytesToFilePath(iconBytes []byte) (string, error) {
 // Loads an image from file and shows it in tray.
 // Shell_NotifyIcon: https://msdn.microsoft.com/en-us/library/windows/desktop/bb762159(v=vs.85).aspx
 func (t *winTray) setIcon(src string) error {
-
 	h, err := t.loadIconFrom(src)
 	if err != nil {
 		return err
@@ -444,7 +441,6 @@ func (t *winTray) setIcon(src string) error {
 // Loads an image from file to be shown in tray or menu item.
 // LoadImage: https://msdn.microsoft.com/en-us/library/windows/desktop/ms648045(v=vs.85).aspx
 func (t *winTray) loadIconFrom(src string) (windows.Handle, error) {
-
 	// Save and reuse handles of loaded images
 	t.muLoadedImages.RLock()
 	h, ok := t.loadedImages[src]

cmd/cmd.go

@@ -12,6 +12,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"math"
 	"net"
 	"net/http"
 	"os"
@@ -19,21 +20,23 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
+	"slices"
 	"strings"
 	"syscall"
 	"time"
 
 	"github.com/containerd/console"
-
+	"github.com/mattn/go-runewidth"
 	"github.com/olekukonko/tablewriter"
 	"github.com/spf13/cobra"
 	"golang.org/x/crypto/ssh"
-	"golang.org/x/exp/slices"
 	"golang.org/x/term"
 
 	"github.com/ollama/ollama/api"
 	"github.com/ollama/ollama/auth"
+	"github.com/ollama/ollama/envconfig"
 	"github.com/ollama/ollama/format"
+	"github.com/ollama/ollama/parser"
 	"github.com/ollama/ollama/progress"
 	"github.com/ollama/ollama/server"
 	"github.com/ollama/ollama/types/errtypes"
@@ -62,7 +65,7 @@ func CreateHandler(cmd *cobra.Command, args []string) error {
 	}
 	defer f.Close()
 
-	modelfile, err := model.ParseFile(f)
+	modelfile, err := parser.ParseFile(f)
 	if err != nil {
 		return err
 	}
@@ -206,7 +209,7 @@ func tempZipFiles(path string) (string, error) {
 		// pytorch files might also be unresolved git lfs references; skip if they are
 		// covers pytorch_model-x-of-y.bin, pytorch_model.fp32-x-of-y.bin, pytorch_model.bin
 		files = append(files, pt...)
-	} else if pt, _ := glob(filepath.Join(path, "consolidated*.pth"), "application/octet-stream"); len(pt) > 0 {
+	} else if pt, _ := glob(filepath.Join(path, "consolidated*.pth"), "application/zip"); len(pt) > 0 {
 		// pytorch files might also be unresolved git lfs references; skip if they are
 		// covers consolidated.x.pth, consolidated.pth
 		files = append(files, pt...)
@@ -324,6 +327,18 @@ func RunHandler(cmd *cobra.Command, args []string) error {
 	}
 	opts.Format = format
 
+	keepAlive, err := cmd.Flags().GetString("keepalive")
+	if err != nil {
+		return err
+	}
+	if keepAlive != "" {
+		d, err := time.ParseDuration(keepAlive)
+		if err != nil {
+			return err
+		}
+		opts.KeepAlive = &api.Duration{Duration: d}
+	}
+
 	prompts := args[1:]
 	// prepend stdin to the prompt if provided
 	if !term.IsTerminal(int(os.Stdin.Fd())) {
@@ -496,6 +511,52 @@ func ListHandler(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
+func ListRunningHandler(cmd *cobra.Command, args []string) error {
+	client, err := api.ClientFromEnvironment()
+	if err != nil {
+		return err
+	}
+
+	models, err := client.ListRunning(cmd.Context())
+	if err != nil {
+		return err
+	}
+
+	var data [][]string
+
+	for _, m := range models.Models {
+		if len(args) == 0 || strings.HasPrefix(m.Name, args[0]) {
+			var procStr string
+			switch {
+			case m.SizeVRAM == 0:
+				procStr = "100% CPU"
+			case m.SizeVRAM == m.Size:
+				procStr = "100% GPU"
+			case m.SizeVRAM > m.Size || m.Size == 0:
+				procStr = "Unknown"
+			default:
+				sizeCPU := m.Size - m.SizeVRAM
+				cpuPercent := math.Round(float64(sizeCPU) / float64(m.Size) * 100)
+				procStr = fmt.Sprintf("%d%%/%d%% CPU/GPU", int(cpuPercent), int(100-cpuPercent))
+			}
+			data = append(data, []string{m.Name, m.Digest[:12], format.HumanBytes(m.Size), procStr, format.HumanTime(m.ExpiresAt, "Never")})
+		}
+	}
+
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetHeader([]string{"NAME", "ID", "SIZE", "PROCESSOR", "UNTIL"})
+	table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
+	table.SetAlignment(tablewriter.ALIGN_LEFT)
+	table.SetHeaderLine(false)
+	table.SetBorder(false)
+	table.SetNoWhiteSpace(true)
+	table.SetTablePadding("\t")
+	table.AppendBulk(data)
+	table.Render()
+
+	return nil
+}
+
 func DeleteHandler(cmd *cobra.Command, args []string) error {
 	client, err := api.ClientFromEnvironment()
 	if err != nil {
@@ -672,6 +733,7 @@ type runOptions struct {
 	Images      []api.ImageData
 	Options     map[string]interface{}
 	MultiModal  bool
+	KeepAlive   *api.Duration
 }
 
 type displayResponseState struct {
@@ -684,7 +746,7 @@ func displayResponse(content string, wordWrap bool, state *displayResponseState)
 	if wordWrap && termWidth >= 10 {
 		for _, ch := range content {
 			if state.lineLength+1 > termWidth-5 {
-				if len(state.wordBuffer) > termWidth-10 {
+				if runewidth.StringWidth(state.wordBuffer) > termWidth-10 {
 					fmt.Printf("%s%c", state.wordBuffer, ch)
 					state.wordBuffer = ""
 					state.lineLength = 0
@@ -692,12 +754,22 @@ func displayResponse(content string, wordWrap bool, state *displayResponseState)
 				}
 
 				// backtrack the length of the last word and clear to the end of the line
-				fmt.Printf("\x1b[%dD\x1b[K\n", len(state.wordBuffer))
+				a := runewidth.StringWidth(state.wordBuffer)
+				if a > 0 {
+					fmt.Printf("\x1b[%dD", a)
+				}
+				fmt.Printf("\x1b[K\n")
 				fmt.Printf("%s%c", state.wordBuffer, ch)
-				state.lineLength = len(state.wordBuffer) + 1
+				chWidth := runewidth.RuneWidth(ch)
+
+				state.lineLength = runewidth.StringWidth(state.wordBuffer) + chWidth
 			} else {
 				fmt.Print(string(ch))
-				state.lineLength += 1
+				state.lineLength += runewidth.RuneWidth(ch)
+				if runewidth.RuneWidth(ch) >= 2 {
+					state.wordBuffer = ""
+					continue
+				}
 
 				switch ch {
 				case ' ':
@@ -766,6 +838,10 @@ func chat(cmd *cobra.Command, opts runOptions) (*api.Message, error) {
 		Options:  opts.Options,
 	}
 
+	if opts.KeepAlive != nil {
+		req.KeepAlive = opts.KeepAlive
+	}
+
 	if err := client.Chat(cancelCtx, req, fn); err != nil {
 		if errors.Is(err, context.Canceled) {
 			return nil, nil
@@ -841,14 +917,15 @@ func generate(cmd *cobra.Command, opts runOptions) error {
 	}
 
 	request := api.GenerateRequest{
-		Model:    opts.Model,
-		Prompt:   opts.Prompt,
-		Context:  generateContext,
-		Images:   opts.Images,
-		Format:   opts.Format,
-		System:   opts.System,
-		Template: opts.Template,
-		Options:  opts.Options,
+		Model:     opts.Model,
+		Prompt:    opts.Prompt,
+		Context:   generateContext,
+		Images:    opts.Images,
+		Format:    opts.Format,
+		System:    opts.System,
+		Template:  opts.Template,
+		Options:   opts.Options,
+		KeepAlive: opts.KeepAlive,
 	}
 
 	if err := client.Generate(ctx, &request, fn); err != nil {
@@ -883,17 +960,11 @@ func generate(cmd *cobra.Command, opts runOptions) error {
 }
 
 func RunServer(cmd *cobra.Command, _ []string) error {
-	// retrieve the OLLAMA_HOST environment variable
-	ollamaHost, err := api.GetOllamaHost()
-	if err != nil {
-		return err
-	}
-
 	if err := initializeKeypair(); err != nil {
 		return err
 	}
 
-	ln, err := net.Listen("tcp", net.JoinHostPort(ollamaHost.Host, ollamaHost.Port))
+	ln, err := net.Listen("tcp", net.JoinHostPort(envconfig.Host.Host, envconfig.Host.Port))
 	if err != nil {
 		return err
 	}
@@ -952,24 +1023,6 @@ func initializeKeypair() error {
 	return nil
 }
 
-//nolint:unused
-func waitForServer(ctx context.Context, client *api.Client) error {
-	// wait for the server to start
-	timeout := time.After(5 * time.Second)
-	tick := time.Tick(500 * time.Millisecond)
-	for {
-		select {
-		case <-timeout:
-			return errors.New("timed out waiting for server to start")
-		case <-tick:
-			if err := client.Heartbeat(ctx); err == nil {
-				return nil // server has started
-			}
-		}
-	}
-
-}
-
 func checkServerHeartbeat(cmd *cobra.Command, _ []string) error {
 	client, err := api.ClientFromEnvironment()
 	if err != nil {
@@ -1006,12 +1059,19 @@ func versionHandler(cmd *cobra.Command, _ []string) {
 	}
 }
 
-func appendHostEnvDocs(cmd *cobra.Command) {
-	const hostEnvDocs = `
+func appendEnvDocs(cmd *cobra.Command, envs []envconfig.EnvVar) {
+	if len(envs) == 0 {
+		return
+	}
+
+	envUsage := `
 Environment Variables:
-      OLLAMA_HOST        The host:port or base URL of the Ollama server (e.g. http://localhost:11434)
 `
-	cmd.SetUsageTemplate(cmd.UsageTemplate() + hostEnvDocs)
+	for _, e := range envs {
+		envUsage += fmt.Sprintf("      %-24s   %s\n", e.Name, e.Description)
+	}
+
+	cmd.SetUsageTemplate(cmd.UsageTemplate() + envUsage)
 }
 
 func NewCLI() *cobra.Command {
@@ -1075,6 +1135,7 @@ func NewCLI() *cobra.Command {
 		RunE:    RunHandler,
 	}
 
+	runCmd.Flags().String("keepalive", "", "Duration to keep a model loaded (e.g. 5m)")
 	runCmd.Flags().Bool("verbose", false, "Show timings for response")
 	runCmd.Flags().Bool("insecure", false, "Use an insecure registry")
 	runCmd.Flags().Bool("nowordwrap", false, "Don't wrap words to the next line automatically")
@@ -1086,15 +1147,6 @@ func NewCLI() *cobra.Command {
 		Args:    cobra.ExactArgs(0),
 		RunE:    RunServer,
 	}
-	serveCmd.SetUsageTemplate(serveCmd.UsageTemplate() + `
-Environment Variables:
-
-    OLLAMA_HOST         The host:port to bind to (default "127.0.0.1:11434")
-    OLLAMA_ORIGINS      A comma separated list of allowed origins.
-    OLLAMA_MODELS       The path to the models directory (default is "~/.ollama/models")
-    OLLAMA_KEEP_ALIVE   The duration that models stay loaded in memory (default is "5m")
-    OLLAMA_DEBUG        Set to 1 to enable additional debug logging
-`)
 
 	pullCmd := &cobra.Command{
 		Use:     "pull MODEL",
@@ -1123,6 +1175,14 @@ Environment Variables:
 		PreRunE: checkServerHeartbeat,
 		RunE:    ListHandler,
 	}
+
+	psCmd := &cobra.Command{
+		Use:     "ps",
+		Short:   "List running models",
+		PreRunE: checkServerHeartbeat,
+		RunE:    ListRunningHandler,
+	}
+
 	copyCmd := &cobra.Command{
 		Use:     "cp SOURCE DESTINATION",
 		Short:   "Copy a model",
@@ -1139,6 +1199,10 @@ Environment Variables:
 		RunE:    DeleteHandler,
 	}
 
+	envVars := envconfig.AsMap()
+
+	envs := []envconfig.EnvVar{envVars["OLLAMA_HOST"]}
+
 	for _, cmd := range []*cobra.Command{
 		createCmd,
 		showCmd,
@@ -1146,10 +1210,33 @@ Environment Variables:
 		pullCmd,
 		pushCmd,
 		listCmd,
+		psCmd,
 		copyCmd,
 		deleteCmd,
+		serveCmd,
 	} {
-		appendHostEnvDocs(cmd)
+		switch cmd {
+		case runCmd:
+			appendEnvDocs(cmd, []envconfig.EnvVar{envVars["OLLAMA_HOST"], envVars["OLLAMA_NOHISTORY"]})
+		case serveCmd:
+			appendEnvDocs(cmd, []envconfig.EnvVar{
+				envVars["OLLAMA_DEBUG"],
+				envVars["OLLAMA_HOST"],
+				envVars["OLLAMA_KEEP_ALIVE"],
+				envVars["OLLAMA_MAX_LOADED_MODELS"],
+				envVars["OLLAMA_MAX_QUEUE"],
+				envVars["OLLAMA_MODELS"],
+				envVars["OLLAMA_NUM_PARALLEL"],
+				envVars["OLLAMA_NOPRUNE"],
+				envVars["OLLAMA_ORIGINS"],
+				envVars["OLLAMA_TMPDIR"],
+				envVars["OLLAMA_FLASH_ATTENTION"],
+				envVars["OLLAMA_LLM_LIBRARY"],
+				envVars["OLLAMA_MAX_VRAM"],
+			})
+		default:
+			appendEnvDocs(cmd, envs)
+		}
 	}
 
 	rootCmd.AddCommand(
@@ -1160,6 +1247,7 @@ Environment Variables:
 		pullCmd,
 		pushCmd,
 		listCmd,
+		psCmd,
 		copyCmd,
 		deleteCmd,
 	)

cmd/interactive.go

@@ -8,15 +8,17 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"slices"
 	"sort"
 	"strings"
 
 	"github.com/spf13/cobra"
-	"golang.org/x/exp/slices"
 
 	"github.com/ollama/ollama/api"
+	"github.com/ollama/ollama/envconfig"
 	"github.com/ollama/ollama/progress"
 	"github.com/ollama/ollama/readline"
+	"github.com/ollama/ollama/types/errtypes"
 )
 
 type MultilineState int
@@ -56,6 +58,11 @@ func loadModel(cmd *cobra.Command, opts *runOptions) error {
 		Model:    opts.Model,
 		Messages: []api.Message{},
 	}
+
+	if opts.KeepAlive != nil {
+		chatReq.KeepAlive = opts.KeepAlive
+	}
+
 	err = client.Chat(cmd.Context(), chatReq, func(resp api.ChatResponse) error {
 		p.StopAndClear()
 		if len(opts.Messages) > 0 {
@@ -132,6 +139,7 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {
 		fmt.Fprintln(os.Stderr, "   Alt + f            Move forward (right) one word")
 		fmt.Fprintln(os.Stderr, "  Ctrl + k            Delete the sentence after the cursor")
 		fmt.Fprintln(os.Stderr, "  Ctrl + u            Delete the sentence before the cursor")
+		fmt.Fprintln(os.Stderr, "  Ctrl + w            Delete the word before the cursor")
 		fmt.Fprintln(os.Stderr, "")
 		fmt.Fprintln(os.Stderr, "  Ctrl + l            Clear the screen")
 		fmt.Fprintln(os.Stderr, "  Ctrl + c            Stop the model from responding")
@@ -176,6 +184,10 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {
 		return err
 	}
 
+	if envconfig.NoHistory {
+		scanner.HistoryDisable()
+	}
+
 	fmt.Print(readline.StartBracketedPaste)
 	defer fmt.Printf(readline.EndBracketedPaste)
 
@@ -276,13 +288,20 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {
 			fn := func(resp api.ProgressResponse) error { return nil }
 			err = client.Create(cmd.Context(), req, fn)
 			if err != nil {
-				fmt.Println("error: couldn't save model")
+				if strings.Contains(err.Error(), errtypes.InvalidModelNameErrMsg) {
+					fmt.Printf("error: The model name '%s' is invalid\n", args[1])
+					continue
+				}
 				return err
 			}
 			fmt.Printf("Created new model '%s'\n", args[1])
 			continue
 		case strings.HasPrefix(line, "/clear"):
 			opts.Messages = []api.Message{}
+			if opts.System != "" {
+				newMessage := api.Message{Role: "system", Content: opts.System}
+				opts.Messages = append(opts.Messages, newMessage)
+			}
 			fmt.Println("Cleared session context")
 			continue
 		case strings.HasPrefix(line, "/set"):

cmd/interactive_test.go

@@ -6,6 +6,7 @@ import (
 	"text/template"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/ollama/ollama/api"
 )
@@ -85,11 +86,11 @@ MESSAGE assistant """Yes it is true, I am half horse, half shark."""
 `
 
 	tmpl, err := template.New("").Parse(expectedModelfile)
-	assert.Nil(t, err)
+	require.NoError(t, err)
 
 	var buf bytes.Buffer
 	err = tmpl.Execute(&buf, opts)
-	assert.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, buf.String(), mf)
 
 	opts.ParentModel = "horseshark"
@@ -107,10 +108,10 @@ MESSAGE assistant """Yes it is true, I am half horse, half shark."""
 `
 
 	tmpl, err = template.New("").Parse(expectedModelfile)
-	assert.Nil(t, err)
+	require.NoError(t, err)
 
 	var parentBuf bytes.Buffer
 	err = tmpl.Execute(&parentBuf, opts)
-	assert.Nil(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, parentBuf.String(), mf)
 }

cmd/start.go

@@ -0,0 +1,27 @@
+//go:build darwin || windows
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"github.com/ollama/ollama/api"
+)
+
+func waitForServer(ctx context.Context, client *api.Client) error {
+	// wait for the server to start
+	timeout := time.After(5 * time.Second)
+	tick := time.Tick(500 * time.Millisecond)
+	for {
+		select {
+		case <-timeout:
+			return errors.New("timed out waiting for server to start")
+		case <-tick:
+			if err := client.Heartbeat(ctx); err == nil {
+				return nil // server has started
+			}
+		}
+	}
+}

convert/convert.go

@@ -18,6 +18,16 @@ import (
 	"github.com/ollama/ollama/llm"
 )
 
+const (
+	_ int32 = iota
+	tokenTypeNormal
+	tokenTypeUnknown
+	tokenTypeControl
+	tokenTypeUserDefined
+	tokenTypeUnused
+	tokenTypeByte
+)
+
 type Params struct {
 	Architectures     []string `json:"architectures"`
 	VocabSize         int      `json:"vocab_size"`
@@ -37,6 +47,8 @@ type Params struct {
 	Experts     int `json:"num_local_experts"`
 	ExpertsUsed int `json:"num_experts_per_tok"`
 
+	PreTokenizer string
+
 	ByteOrder
 }
 
@@ -74,10 +86,9 @@ func GetModelFormat(dirname string) (ModelFormat, error) {
 	}
 
 	for _, fn := range files {
-		slog.Debug(fmt.Sprintf("file = %s", fn))
 		if strings.HasSuffix(fn, ".safetensors") {
 			return &SafetensorFormat{}, nil
-		} else if strings.HasSuffix(fn, ".bin") {
+		} else if strings.HasSuffix(fn, ".bin") || strings.HasSuffix(fn, ".pth") {
 			slog.Debug("model is torch")
 			return &TorchFormat{}, nil
 		}
@@ -92,6 +103,7 @@ type Vocab struct {
 	Tokens []string
 	Scores []float32
 	Types  []int32
+	Merges []string
 }
 
 func LoadSentencePieceTokens(dirpath string, params *Params) (*Vocab, error) {
@@ -170,17 +182,17 @@ func LoadSentencePieceTokens(dirpath string, params *Params) (*Vocab, error) {
 		}
 		v.Tokens = append(v.Tokens, t.key)
 		v.Scores = append(v.Scores, -1000.0)
-		v.Types = append(v.Types, int32(llm.GGUFTokenUserDefined))
+		v.Types = append(v.Types, tokenTypeUserDefined)
 	}
 	slog.Info(fmt.Sprintf("vocab size w/ extra tokens: %d", len(v.Tokens)))
 
 	if params.VocabSize > len(v.Tokens) {
 		missingTokens := params.VocabSize - len(v.Tokens)
 		slog.Warn(fmt.Sprintf("vocab is missing %d tokens", missingTokens))
-		for cnt := 0; cnt < missingTokens; cnt++ {
+		for cnt := range missingTokens {
 			v.Tokens = append(v.Tokens, fmt.Sprintf("<dummy%05d>", cnt+1))
 			v.Scores = append(v.Scores, -1)
-			v.Types = append(v.Types, int32(llm.GGUFTokenUserDefined))
+			v.Types = append(v.Types, tokenTypeUserDefined)
 		}
 	}
 

convert/convert_test.go

@@ -0,0 +1,103 @@
+//go:build slow
+
+package convert
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/ollama/ollama/llm"
+)
+
+func convertFull(t *testing.T, p string) (llm.KV, llm.Tensors) {
+	t.Helper()
+
+	mf, err := GetModelFormat(p)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	params, err := mf.GetParams(p)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	arch, err := mf.GetModelArch("", p, params)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := arch.LoadVocab(); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := arch.GetTensors(); err != nil {
+		t.Fatal(err)
+	}
+
+	f, err := os.CreateTemp(t.TempDir(), "f16")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+
+	if err := arch.WriteGGUF(f); err != nil {
+		t.Fatal(err)
+	}
+
+	r, err := os.Open(f.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer r.Close()
+
+	m, _, err := llm.DecodeGGML(r)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return m.KV(), m.Tensors()
+}
+
+func TestConvertFull(t *testing.T) {
+	cases := []struct {
+		path    string
+		arch    string
+		tensors int
+		layers  int
+	}{
+		{"Meta-Llama-3-8B-Instruct", "llama", 291, 35},
+		{"Mistral-7B-Instruct-v0.2", "llama", 291, 35},
+		{"Mixtral-8x7B-Instruct-v0.1", "llama", 291, 35},
+		{"gemma-2b-it", "gemma", 164, 20},
+	}
+
+	for _, tt := range cases {
+		t.Run(tt.path, func(t *testing.T) {
+			p := filepath.Join("testdata", tt.path)
+			if _, err := os.Stat(p); err != nil {
+				t.Skipf("%s not found", p)
+			}
+
+			kv, tensors := convertFull(t, p)
+
+			if kv.Architecture() != tt.arch {
+				t.Fatalf("expected llama, got %s", kv.Architecture())
+			}
+
+			if kv.FileType().String() != "F16" {
+				t.Fatalf("expected F16, got %s", kv.FileType())
+			}
+
+			if len(tensors) != tt.tensors {
+				t.Fatalf("expected %d tensors, got %d", tt.tensors, len(tensors))
+			}
+
+			layers := tensors.Layers()
+			if len(layers) != tt.layers {
+				t.Fatalf("expected %d layers, got %d", tt.layers, len(layers))
+			}
+		})
+	}
+}

convert/gemma.go

@@ -1,14 +1,11 @@
 package convert
 
 import (
-	"encoding/binary"
 	"fmt"
 	"io"
 	"log/slog"
-	"os"
 	"strings"
 
-	"github.com/d4l3k/go-bfloat16"
 	"github.com/pdevine/tensor"
 	"github.com/pdevine/tensor/native"
 
@@ -19,49 +16,26 @@ type GemmaModel struct {
 	ModelData
 }
 
-func gemmaLayerHandler(w io.Writer, r safetensorWriterTo, f *os.File) error {
-	slog.Debug(fmt.Sprintf("converting '%s'", r.t.Name))
-
-	data := make([]byte, r.end-r.start)
-	if err := binary.Read(f, r.bo, data); err != nil {
-		return err
-	}
-
-	tDataF32 := bfloat16.DecodeFloat32(data)
-
-	var err error
-	tDataF32, err = addOnes(tDataF32, int(r.t.Shape[0]))
-	if err != nil {
-		return err
-	}
-
-	if err := binary.Write(w, r.bo, tDataF32); err != nil {
-		return err
-	}
-	return nil
-}
-
 func addOnes(data []float32, vectorSize int) ([]float32, error) {
 	n := tensor.New(tensor.WithShape(vectorSize), tensor.WithBacking(data))
 	ones := tensor.Ones(tensor.Float32, vectorSize)
 
-	var err error
-	n, err = n.Add(ones)
+	n, err := n.Add(ones)
 	if err != nil {
-		return []float32{}, err
+		return nil, err
 	}
 
-	newN, err := native.SelectF32(n, 0)
+	ts, err := native.SelectF32(n, 0)
 	if err != nil {
-		return []float32{}, err
+		return nil, err
 	}
 
-	var fullTensor []float32
-	for _, v := range newN {
-		fullTensor = append(fullTensor, v...)
+	var f32s []float32
+	for _, t := range ts {
+		f32s = append(f32s, t...)
 	}
 
-	return fullTensor, nil
+	return f32s, nil
 }
 
 func (m *GemmaModel) GetTensors() error {
@@ -71,12 +45,10 @@ func (m *GemmaModel) GetTensors() error {
 	}
 
 	slog.Debug(fmt.Sprintf("Total tensors: %d", len(t)))
-
-	m.Tensors = []llm.Tensor{}
 	for _, l := range t {
 		if strings.HasSuffix(l.Name, "norm.weight") {
 			wt := l.WriterTo.(safetensorWriterTo)
-			wt.handler = gemmaLayerHandler
+			wt.repacker = m.Repack
 			l.WriterTo = wt
 		}
 		m.Tensors = append(m.Tensors, l)
@@ -94,6 +66,10 @@ func (m *GemmaModel) LoadVocab() error {
 	return nil
 }
 
+func (m *GemmaModel) Repack(_ string, data []float32, shape []uint64) ([]float32, error) {
+	return addOnes(data, int(shape[0]))
+}
+
 func (m *GemmaModel) WriteGGUF(ws io.WriteSeeker) error {
 	kv := llm.KV{
 		"general.architecture":                   "gemma",

convert/llama.go

@@ -1,17 +1,17 @@
 package convert
 
 import (
-	"encoding/binary"
+	"cmp"
+	"errors"
 	"fmt"
 	"io"
-	"log/slog"
+	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
 
-	"github.com/nlpodyssey/gopickle/pytorch"
 	"github.com/pdevine/tensor"
 	"github.com/pdevine/tensor/native"
-	"github.com/x448/float16"
 
 	"github.com/ollama/ollama/llm"
 )
@@ -20,81 +20,12 @@ type LlamaModel struct {
 	ModelData
 }
 
-func llamaLayerHandler(w io.Writer, r torchWriterTo) error {
-	slog.Debug(fmt.Sprintf("repacking layer '%s'", r.t.Name))
-
-	data := r.storage.(*pytorch.HalfStorage).Data
-	tData := make([]uint16, len(data))
-	for cnt, v := range data {
-		tData[cnt] = uint16(float16.Fromfloat32(v))
-	}
-
-	var err error
-	var heads uint32
-	if strings.Contains(r.t.Name, "attn_q") {
-		heads = uint32(r.params.AttentionHeads)
-	} else if strings.Contains(r.t.Name, "attn_k") {
-		heads = uint32(r.params.KeyValHeads)
-		if heads == 0 {
-			heads = uint32(r.params.AttentionHeads)
-		}
-	} else {
-		return fmt.Errorf("unknown layer type")
-	}
-
-	slog.Debug(fmt.Sprintf("heads = %d", heads))
-
-	tData, err = llamaRepack(tData, int(heads), r.t.Shape)
-	if err != nil {
-		return err
-	}
-
-	if err = binary.Write(w, r.bo, tData); err != nil {
-		return err
-	}
-	return nil
-}
-
-func llamaRepack(data []uint16, heads int, shape []uint64) ([]uint16, error) {
-	n := tensor.New(tensor.WithShape(int(shape[0]), int(shape[1])), tensor.WithBacking(data))
-	origShape := n.Shape().Clone()
-
-	// reshape the tensor and swap axes 1 and 2 to unpack the layer for gguf
-	if err := n.Reshape(heads, 2, origShape[0]/heads/2, origShape[1]); err != nil {
-		return nil, err
-	}
-
-	if err := n.T(0, 2, 1, 3); err != nil {
-		return nil, err
-	}
-
-	if err := n.Reshape(origShape...); err != nil {
-		return nil, err
-	}
-
-	if err := n.Transpose(); err != nil {
-		return nil, err
-	}
-	newN, err := native.SelectU16(n, 1)
-	if err != nil {
-		return nil, err
-	}
-
-	var fullTensor []uint16
-	for _, v := range newN {
-		fullTensor = append(fullTensor, v...)
-	}
-	return fullTensor, nil
-}
-
 func (m *LlamaModel) GetTensors() error {
 	t, err := m.Format.GetTensors(m.Path, m.Params)
 	if err != nil {
 		return err
 	}
 
-	m.Tensors = []llm.Tensor{}
-
 	pattern := `^blk\.[0-9]+\.attn_(?P<layer>q|k)\.weight$`
 	re, err := regexp.Compile(pattern)
 	if err != nil {
@@ -104,10 +35,16 @@ func (m *LlamaModel) GetTensors() error {
 	for _, l := range t {
 		matches := re.FindAllStringSubmatch(l.Name, -1)
 		if len(matches) > 0 {
-			slog.Debug(fmt.Sprintf("setting handler for: %s", l.Name))
-			wt := l.WriterTo.(torchWriterTo)
-			wt.handler = llamaLayerHandler
-			l.WriterTo = wt
+			switch m.Format.(type) {
+			case *TorchFormat:
+				wt := l.WriterTo.(torchWriterTo)
+				wt.repacker = m.Repack
+				l.WriterTo = wt
+			case *SafetensorFormat:
+				wt := l.WriterTo.(safetensorWriterTo)
+				wt.repacker = m.Repack
+				l.WriterTo = wt
+			}
 		}
 		m.Tensors = append(m.Tensors, l)
 	}
@@ -115,19 +52,22 @@ func (m *LlamaModel) GetTensors() error {
 	return nil
 }
 
-func (m *LlamaModel) LoadVocab() error {
-	var v *Vocab
-	var err error
-
-	slog.Debug("loading vocab")
-	v, err = LoadSentencePieceTokens(m.Path, m.Params)
-	if err != nil {
+func (m *LlamaModel) LoadVocab() (err error) {
+	pre, ts, merges, err := parseTokens(filepath.Join(m.Path, "tokenizer.json"))
+	if errors.Is(err, os.ErrNotExist) {
+		return nil
+	} else if err != nil {
 		return err
 	}
 
-	slog.Debug("vocab loaded")
+	m.Vocab = &Vocab{}
+	for _, t := range ts {
+		m.Vocab.Tokens = append(m.Vocab.Tokens, t.Content)
+		m.Vocab.Types = append(m.Vocab.Types, t.Type())
+	}
 
-	m.Vocab = v
+	m.Vocab.Merges = merges
+	m.Params.PreTokenizer = pre
 	return nil
 }
 
@@ -140,23 +80,80 @@ func (m *LlamaModel) WriteGGUF(ws io.WriteSeeker) error {
 		"llama.embedding_length":                 uint32(m.Params.HiddenSize),
 		"llama.block_count":                      uint32(m.Params.HiddenLayers),
 		"llama.feed_forward_length":              uint32(m.Params.IntermediateSize),
+		"llama.rope.freq_base":                   float32(m.Params.RopeFrequencyBase),
 		"llama.rope.dimension_count":             uint32(m.Params.HiddenSize / m.Params.AttentionHeads),
 		"llama.attention.head_count":             uint32(m.Params.AttentionHeads),
 		"llama.attention.head_count_kv":          uint32(m.Params.KeyValHeads),
 		"llama.attention.layer_norm_rms_epsilon": float32(m.Params.NormEPS),
 		"general.file_type":                      uint32(1),
-		"tokenizer.ggml.model":                   "llama",
+		"tokenizer.ggml.model":                   "gpt2",
 
+		"tokenizer.ggml.pre":        m.Params.PreTokenizer,
 		"tokenizer.ggml.tokens":     m.Vocab.Tokens,
-		"tokenizer.ggml.scores":     m.Vocab.Scores,
 		"tokenizer.ggml.token_type": m.Vocab.Types,
 
 		"tokenizer.ggml.bos_token_id":     uint32(m.Params.BoSTokenID),
 		"tokenizer.ggml.eos_token_id":     uint32(m.Params.EoSTokenID),
 		"tokenizer.ggml.unknown_token_id": uint32(0),
-		"tokenizer.ggml.add_bos_token":    true,
-		"tokenizer.ggml.add_eos_token":    false,
+	}
+
+	if len(m.Vocab.Merges) > 0 {
+		kv["tokenizer.ggml.merges"] = m.Vocab.Merges
+	} else {
+		kv["tokenizer.ggml.scores"] = m.Vocab.Scores
 	}
 
 	return llm.NewGGUFV3(m.Params.ByteOrder).Encode(ws, kv, m.Tensors)
 }
+
+func (m *LlamaModel) Repack(name string, data []float32, shape []uint64) ([]float32, error) {
+	return llamaRepack(name, m.Params, data, shape)
+}
+
+func llamaRepack(name string, params *Params, data []float32, shape []uint64) ([]float32, error) {
+	var dims []int
+	for _, dim := range shape {
+		if dim != 0 {
+			dims = append(dims, int(dim))
+		}
+	}
+
+	var heads int
+	switch {
+	case strings.HasSuffix(name, "attn_q.weight"):
+		heads = params.AttentionHeads
+	case strings.HasSuffix(name, "attn_k.weight"):
+		heads = cmp.Or(params.KeyValHeads, params.AttentionHeads)
+	default:
+		return nil, fmt.Errorf("unknown tensor name: %s", name)
+	}
+
+	n := tensor.New(tensor.WithShape(dims...), tensor.WithBacking(data))
+	if err := n.Reshape(append([]int{heads, 2, dims[0] / heads / 2}, dims[1:]...)...); err != nil {
+		return nil, err
+	}
+
+	if err := n.T(0, 2, 1, 3); err != nil {
+		return nil, err
+	}
+
+	if err := n.Reshape(dims...); err != nil {
+		return nil, err
+	}
+
+	if err := n.Transpose(); err != nil {
+		return nil, err
+	}
+
+	ts, err := native.SelectF32(n, 1)
+	if err != nil {
+		return nil, err
+	}
+
+	var f32s []float32
+	for _, t := range ts {
+		f32s = append(f32s, t...)
+	}
+
+	return f32s, nil
+}

convert/mistral.go

@@ -1,17 +1,8 @@
 package convert
 
 import (
-	"encoding/binary"
-	"fmt"
 	"io"
-	"os"
 	"regexp"
-	"strings"
-
-	"github.com/d4l3k/go-bfloat16"
-	"github.com/pdevine/tensor"
-	"github.com/pdevine/tensor/native"
-	"github.com/x448/float16"
 
 	"github.com/ollama/ollama/llm"
 )
@@ -20,90 +11,12 @@ type MistralModel struct {
 	ModelData
 }
 
-func mistralLayerHandler(w io.Writer, r safetensorWriterTo, f *os.File) error {
-	layerSize := r.end - r.start
-
-	var err error
-	tData := make([]uint16, layerSize/2)
-	if err = binary.Read(f, r.bo, tData); err != nil {
-		return err
-	}
-
-	var heads uint32
-	if strings.Contains(r.t.Name, "attn_q") {
-		heads = uint32(r.params.AttentionHeads)
-	} else if strings.Contains(r.t.Name, "attn_k") {
-		heads = uint32(r.params.KeyValHeads)
-		if heads == 0 {
-			heads = uint32(r.params.AttentionHeads)
-		}
-	} else {
-		return fmt.Errorf("unknown layer type")
-	}
-
-	tData, err = repack(tData, int(heads), r.t.Shape)
-	if err != nil {
-		return err
-	}
-
-	var buf []byte
-	for _, n := range tData {
-		buf = r.bo.AppendUint16(buf, n)
-	}
-
-	tempBuf := make([]uint16, len(tData))
-	tDataF32 := bfloat16.DecodeFloat32(buf)
-	for cnt, v := range tDataF32 {
-		tDataF16 := float16.Fromfloat32(v)
-		tempBuf[cnt] = uint16(tDataF16)
-	}
-
-	if err = binary.Write(w, r.bo, tempBuf); err != nil {
-		return err
-	}
-	return nil
-}
-
-func repack(data []uint16, heads int, shape []uint64) ([]uint16, error) {
-	n := tensor.New(tensor.WithShape(int(shape[0]), int(shape[1])), tensor.WithBacking(data))
-	origShape := n.Shape().Clone()
-
-	// reshape the tensor and swap axes 1 and 2 to unpack the layer for gguf
-	if err := n.Reshape(heads, 2, origShape[0]/heads/2, origShape[1]); err != nil {
-		return nil, err
-	}
-
-	if err := n.T(0, 2, 1, 3); err != nil {
-		return nil, err
-	}
-
-	if err := n.Reshape(origShape...); err != nil {
-		return nil, err
-	}
-
-	if err := n.Transpose(); err != nil {
-		return nil, err
-	}
-	newN, err := native.SelectU16(n, 1)
-	if err != nil {
-		return nil, err
-	}
-
-	var fullTensor []uint16
-	for _, v := range newN {
-		fullTensor = append(fullTensor, v...)
-	}
-	return fullTensor, nil
-}
-
 func (m *MistralModel) GetTensors() error {
 	t, err := m.Format.GetTensors(m.Path, m.Params)
 	if err != nil {
 		return err
 	}
 
-	m.Tensors = []llm.Tensor{}
-
 	pattern := `^blk\.[0-9]+\.attn_(?P<layer>q|k)\.weight$`
 	re, err := regexp.Compile(pattern)
 	if err != nil {
@@ -114,7 +27,7 @@ func (m *MistralModel) GetTensors() error {
 		matches := re.FindAllStringSubmatch(l.Name, -1)
 		if len(matches) > 0 {
 			wt := l.WriterTo.(safetensorWriterTo)
-			wt.handler = mistralLayerHandler
+			wt.repacker = m.Repack
 			l.WriterTo = wt
 		}
 		m.Tensors = append(m.Tensors, l)
@@ -160,3 +73,7 @@ func (m *MistralModel) WriteGGUF(ws io.WriteSeeker) error {
 
 	return llm.NewGGUFV3(m.Params.ByteOrder).Encode(ws, kv, m.Tensors)
 }
+
+func (m *MistralModel) Repack(name string, data []float32, shape []uint64) ([]float32, error) {
+	return llamaRepack(name, m.Params, data, shape)
+}

convert/mixtral.go

@@ -17,8 +17,6 @@ func (m *MixtralModel) GetTensors() error {
 		return err
 	}
 
-	m.Tensors = []llm.Tensor{}
-
 	pattern := `^blk\.[0-9]+\.attn_(?P<layer>q|k)\.weight$`
 	re, err := regexp.Compile(pattern)
 	if err != nil {
@@ -29,7 +27,7 @@ func (m *MixtralModel) GetTensors() error {
 		matches := re.FindAllStringSubmatch(l.Name, -1)
 		if len(matches) > 0 {
 			wt := l.WriterTo.(safetensorWriterTo)
-			wt.handler = mistralLayerHandler
+			wt.repacker = m.Repack
 			l.WriterTo = wt
 		}
 		m.Tensors = append(m.Tensors, l)
@@ -83,3 +81,7 @@ func (m *MixtralModel) WriteGGUF(ws io.WriteSeeker) error {
 
 	return llm.NewGGUFV3(m.Params.ByteOrder).Encode(ws, kv, m.Tensors)
 }
+
+func (m *MixtralModel) Repack(name string, data []float32, shape []uint64) ([]float32, error) {
+	return llamaRepack(name, m.Params, data, shape)
+}

convert/safetensors.go

@@ -6,14 +6,13 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"log/slog"
 	"os"
 	"path/filepath"
 	"regexp"
 	"slices"
+	"strings"
 
 	"github.com/d4l3k/go-bfloat16"
-	"github.com/mitchellh/mapstructure"
 	"github.com/x448/float16"
 
 	"github.com/ollama/ollama/llm"
@@ -26,39 +25,38 @@ type safetensorWriterTo struct {
 	bo     ByteOrder
 
 	filename string
+	dtype    string
 
-	start, end, padding uint64
-	handler             func(w io.Writer, r safetensorWriterTo, f *os.File) error
+	offset, size int64
+	repacker     func(string, []float32, []uint64) ([]float32, error)
 }
 
-type tensorMetaData struct {
-	Type    string `mapstructure:"dtype"`
-	Shape   []int  `mapstructure:"shape"`
-	Offsets []int  `mapstructure:"data_offsets"`
+type safetensorMetadata struct {
+	Type    string   `json:"dtype"`
+	Shape   []uint64 `json:"shape"`
+	Offsets []int64  `json:"data_offsets"`
 }
 
 type SafetensorFormat struct{}
 
 func (m *SafetensorFormat) GetTensors(dirpath string, params *Params) ([]llm.Tensor, error) {
-	slog.Debug("getting tensor data")
 	var tensors []llm.Tensor
-	files, err := filepath.Glob(filepath.Join(dirpath, "/model-*.safetensors"))
+	matches, err := filepath.Glob(filepath.Join(dirpath, "*.safetensors"))
 	if err != nil {
 		return nil, err
 	}
 
 	var offset uint64
-	for _, f := range files {
+	for _, f := range matches {
 		var t []llm.Tensor
 		var err error
 		t, offset, err = m.readTensors(f, offset, params)
 		if err != nil {
-			slog.Error(err.Error())
 			return nil, err
 		}
+
 		tensors = append(tensors, t...)
 	}
-	slog.Debug(fmt.Sprintf("all tensors = %d", len(tensors)))
 	return tensors, nil
 }
 
@@ -69,73 +67,60 @@ func (m *SafetensorFormat) readTensors(fn string, offset uint64, params *Params)
 	}
 	defer f.Close()
 
-	var jsonSize uint64
-	if err := binary.Read(f, binary.LittleEndian, &jsonSize); err != nil {
+	var n int64
+	if err := binary.Read(f, binary.LittleEndian, &n); err != nil {
 		return nil, 0, err
 	}
 
-	buf := make([]byte, jsonSize)
-	_, err = io.ReadFull(f, buf)
-	if err != nil {
+	b := bytes.NewBuffer(make([]byte, 0, n))
+	if _, err = io.CopyN(b, f, n); err != nil {
 		return nil, 0, err
 	}
 
-	d := json.NewDecoder(bytes.NewBuffer(buf))
-	d.UseNumber()
-	var parsed map[string]interface{}
-	if err = d.Decode(&parsed); err != nil {
+	var headers map[string]safetensorMetadata
+	if err := json.NewDecoder(b).Decode(&headers); err != nil {
 		return nil, 0, err
 	}
 
 	var keys []string
-	for k := range parsed {
-		keys = append(keys, k)
+	for key := range headers {
+		if !strings.HasSuffix(key, "self_attn.rotary_embd.inv_freq") {
+			keys = append(keys, key)
+		}
 	}
 
 	slices.Sort(keys)
-	slog.Info("converting layers")
 
 	var tensors []llm.Tensor
-	for _, k := range keys {
-		vals := parsed[k].(map[string]interface{})
-		var data tensorMetaData
-		if err = mapstructure.Decode(vals, &data); err != nil {
-			slog.Error("couldn't decode properly")
-			return nil, 0, err
-		}
+	for _, key := range keys {
+		value := headers[key]
 
-		var size uint64
 		var kind uint32
-		switch len(data.Shape) {
+		switch len(value.Shape) {
 		case 0:
-			// metadata
+			// valuedata
 			continue
-		case 1:
-			// convert to float32
-			kind = 0
-			size = uint64(data.Shape[0] * 4)
 		case 2:
-			// convert to float16
 			kind = 1
-			size = uint64(data.Shape[0] * data.Shape[1] * 2)
 		}
 
-		ggufName, err := m.GetLayerName(k)
+		name, err := m.GetLayerName(key)
 		if err != nil {
-			slog.Error(err.Error())
 			return nil, 0, err
 		}
 
-		shape := []uint64{0, 0, 0, 0}
-		for i := range data.Shape {
-			shape[i] = uint64(data.Shape[i])
+		shape := make([]uint64, len(value.Shape))
+		copy(shape, value.Shape)
+
+		pad := func(s int64) int64 {
+			return 8 + n + s
 		}
 
 		t := llm.Tensor{
-			Name:   ggufName,
+			Name:   name,
 			Kind:   kind,
 			Offset: offset,
-			Shape:  shape[:],
+			Shape:  shape,
 		}
 
 		t.WriterTo = safetensorWriterTo{
@@ -143,18 +128,15 @@ func (m *SafetensorFormat) readTensors(fn string, offset uint64, params *Params)
 			params:   params,
 			bo:       params.ByteOrder,
 			filename: fn,
-			start:    uint64(data.Offsets[0]),
-			end:      uint64(data.Offsets[1]),
-			padding:  8 + jsonSize,
+			dtype:    value.Type,
+			offset:   pad(value.Offsets[0]),
+			size:     pad(value.Offsets[1]) - pad(value.Offsets[0]),
 		}
 
-		offset += size
+		offset += t.Size()
 		tensors = append(tensors, t)
 	}
 
-	slog.Debug(fmt.Sprintf("total tensors for file = %d", len(tensors)))
-	slog.Debug(fmt.Sprintf("offset = %d", offset))
-
 	return tensors, offset, nil
 }
 
@@ -167,9 +149,7 @@ func (m *SafetensorFormat) GetParams(dirpath string) (*Params, error) {
 
 	var params Params
 
-	d := json.NewDecoder(f)
-	err = d.Decode(&params)
-	if err != nil {
+	if err := json.NewDecoder(f).Decode(&params); err != nil {
 		return nil, err
 	}
 
@@ -224,55 +204,58 @@ func (r safetensorWriterTo) WriteTo(w io.Writer) (n int64, err error) {
 	}
 	defer f.Close()
 
-	if _, err = f.Seek(int64(r.padding+r.start), 0); err != nil {
+	if _, err = f.Seek(r.offset, io.SeekStart); err != nil {
 		return 0, err
 	}
 
-	// use the handler if one is present
-	if r.handler != nil {
-		return 0, r.handler(w, r, f)
-	}
-
-	remaining := r.end - r.start
-
-	bufSize := uint64(10240)
-	var finished bool
-	for {
-		data := make([]byte, min(bufSize, remaining))
-
-		b, err := io.ReadFull(f, data)
-		remaining -= uint64(b)
-
-		if err == io.EOF || remaining <= 0 {
-			finished = true
-		} else if err != nil {
+	var f32s []float32
+	switch r.dtype {
+	case "F32":
+		f32s = make([]float32, r.size/4)
+		if err = binary.Read(f, r.bo, f32s); err != nil {
+			return 0, err
+		}
+	case "F16":
+		u16s := make([]uint16, r.size/2)
+		if err = binary.Read(f, r.bo, u16s); err != nil {
 			return 0, err
 		}
 
-		// convert bfloat16 -> ieee float32
-		tDataF32 := bfloat16.DecodeFloat32(data)
-
-		switch r.t.Kind {
-		case 0:
-			if err := binary.Write(w, r.bo, tDataF32); err != nil {
-				return 0, err
-			}
-		case 1:
-			// convert float32 -> float16
-			tempBuf := make([]uint16, len(data)/2)
-			for cnt, v := range tDataF32 {
-				tDataF16 := float16.Fromfloat32(v)
-				tempBuf[cnt] = uint16(tDataF16)
-			}
-			if err := binary.Write(w, r.bo, tempBuf); err != nil {
-				return 0, err
-			}
+		for _, b := range u16s {
+			f32s = append(f32s, float16.Frombits(b).Float32())
 		}
-		if finished {
-			break
+
+	case "BF16":
+		u8s := make([]uint8, r.size)
+		if err = binary.Read(f, r.bo, u8s); err != nil {
+			return 0, err
+		}
+
+		f32s = bfloat16.DecodeFloat32(u8s)
+	default:
+		return 0, fmt.Errorf("unknown data type: %s", r.dtype)
+	}
+
+	if r.repacker != nil {
+		f32s, err = r.repacker(r.t.Name, f32s, r.t.Shape)
+		if err != nil {
+			return 0, err
 		}
 	}
-	return 0, nil
+
+	switch r.t.Kind {
+	case 0:
+		return 0, binary.Write(w, r.bo, f32s)
+	case 1:
+		f16s := make([]uint16, len(f32s))
+		for i := range f32s {
+			f16s[i] = float16.Fromfloat32(f32s[i]).Bits()
+		}
+
+		return 0, binary.Write(w, r.bo, f16s)
+	default:
+		return 0, fmt.Errorf("unknown storage type: %d", r.t.Kind)
+	}
 }
 
 func (m *SafetensorFormat) GetModelArch(name, dirPath string, params *Params) (ModelArch, error) {
@@ -281,6 +264,15 @@ func (m *SafetensorFormat) GetModelArch(name, dirPath string, params *Params) (M
 		return nil, fmt.Errorf("No architecture specified to convert")
 	case 1:
 		switch params.Architectures[0] {
+		case "LlamaForCausalLM":
+			return &LlamaModel{
+				ModelData{
+					Name:   name,
+					Path:   dirPath,
+					Params: params,
+					Format: m,
+				},
+			}, nil
 		case "MistralForCausalLM":
 			return &MistralModel{
 				ModelData{

convert/tokenizer.go

@@ -0,0 +1,106 @@
+package convert
+
+import (
+	"cmp"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"os"
+	"slices"
+
+	"golang.org/x/exp/maps"
+)
+
+type Tokenizer struct {
+	Version     string         `json:"version"`
+	AddedTokens []Token        `json:"added_tokens"`
+	Model       TokenizerModel `json:"model"`
+
+	PreTokenizer struct {
+		PreTokenizers []struct {
+			Type    string `json:"type"`
+			Pattern struct {
+				Regex string `json:"Regex"`
+			} `json:"pattern"`
+		} `json:"pretokenizers"`
+	} `json:"pre_tokenizer"`
+}
+
+type TokenizerModel struct {
+	Type   string         `json:"type"`
+	Vocab  map[string]int `json:"vocab"`
+	Merges []string       `json:"merges"`
+	Tokens []Token
+}
+
+type Token struct {
+	ID          int    `json:"id"`
+	Content     string `json:"content"`
+	Special     bool   `json:"special"`
+	UserDefined bool
+}
+
+func (t *Token) Type() int32 {
+	switch {
+	case t.Special:
+		return tokenTypeControl
+	case t.UserDefined:
+		return tokenTypeUserDefined
+	default:
+		return tokenTypeNormal
+	}
+}
+
+func (t *Tokenizer) maxID() int {
+	return max(
+		slices.Max(maps.Values(t.Model.Vocab)),
+		slices.MaxFunc(t.AddedTokens, func(a, b Token) int {
+			return cmp.Compare(a.ID, b.ID)
+		}).ID,
+	)
+}
+
+func parseTokens(dirpath string) (pre string, tokens []Token, merges []string, err error) {
+	f, err := os.Open(dirpath)
+	if err != nil {
+		panic(err)
+	}
+	defer f.Close()
+
+	var t Tokenizer
+	if err := json.NewDecoder(f).Decode(&t); err != nil {
+		return "", nil, nil, err
+	}
+
+	tokens = make([]Token, t.maxID()+1)
+	for k, v := range t.Model.Vocab {
+		tokens[v] = Token{ID: v, Content: k, Special: false, UserDefined: false}
+	}
+
+	for _, v := range t.AddedTokens {
+		v.UserDefined = true
+		tokens[v.ID] = v
+	}
+
+	sha256sum := sha256.New()
+	for _, pt := range t.PreTokenizer.PreTokenizers {
+		if pt.Type == "Split" && pt.Pattern.Regex != "" {
+			sha256sum.Write([]byte(pt.Pattern.Regex))
+		}
+	}
+
+	switch digest := fmt.Sprintf("%x", sha256sum.Sum(nil)); digest {
+	case "d98f9631be1e9607a9848c26c1f9eac1aa9fc21ac6ba82a2fc0741af9780a48f":
+		pre = "llama-bpe"
+	case "03df5c5863ad70781dcfdef491ead25140f895fe8010964be0daefe27be32b02":
+		pre = "deepseek-llm"
+	case "21cde974d587f0d54dc8d56b183cc1e6239600172035c68fbd6d4b9f8da0576e":
+		pre = "deepseek-coder"
+	default:
+		slog.Warn("unknown pretokenizer, using default", "digest", digest)
+		pre = "default"
+	}
+
+	return pre, tokens, t.Model.Merges, nil
+}

convert/torch.go

@@ -24,8 +24,8 @@ type torchWriterTo struct {
 	params *Params
 	bo     ByteOrder
 
-	storage pytorch.StorageInterface
-	handler func(w io.Writer, r torchWriterTo) error
+	storage  pytorch.StorageInterface
+	repacker func(string, []float32, []uint64) ([]float32, error)
 }
 
 type TorchFormat struct{}
@@ -33,14 +33,14 @@ type TorchFormat struct{}
 func (tf *TorchFormat) GetTensors(dirpath string, params *Params) ([]llm.Tensor, error) {
 	slog.Debug("getting torch tensors")
 
-	files, err := filepath.Glob(filepath.Join(dirpath, "pytorch_model-*.bin"))
-	if err != nil {
-		slog.Error("didn't find any torch files")
-		return nil, err
+	var files []string
+	if pt, _ := filepath.Glob(filepath.Join(dirpath, "consolidated*.pth")); len(pt) > 0 {
+		files = append(files, pt...)
+	} else if pt, _ := filepath.Glob(filepath.Join(dirpath, "pytorch_model*.pth")); len(pt) > 0 {
+		files = append(files, pt...)
 	}
 
 	var offset uint64
-
 	var tensors []llm.Tensor
 	for _, fn := range files {
 		m, err := pytorch.Load(fn)
@@ -77,7 +77,7 @@ func (tf *TorchFormat) GetTensors(dirpath string, params *Params) ([]llm.Tensor,
 				slog.Error(err.Error())
 				return nil, err
 			}
-			slog.Debug(fmt.Sprintf("finding name for '%s' -> '%s'", k.(string), ggufName))
+			slog.Debug(fmt.Sprintf("'%35s': '%30s' %10d [%#v]", k.(string), ggufName, size, tshape))
 
 			shape := []uint64{0, 0, 0, 0}
 			for i := range tshape {
@@ -88,7 +88,7 @@ func (tf *TorchFormat) GetTensors(dirpath string, params *Params) ([]llm.Tensor,
 				Name:   ggufName,
 				Kind:   kind,
 				Offset: offset, // calculate the offset
-				Shape:  shape[:],
+				Shape:  shape,
 			}
 
 			tensor.WriterTo = torchWriterTo{
@@ -104,7 +104,6 @@ func (tf *TorchFormat) GetTensors(dirpath string, params *Params) ([]llm.Tensor,
 	}
 
 	return tensors, nil
-
 }
 
 func getAltParams(dirpath string) (*Params, error) {
@@ -120,7 +119,7 @@ func getAltParams(dirpath string) (*Params, error) {
 		AttentionHeads int     `json:"n_heads"`
 		KeyValHeads    int     `json:"n_kv_heads"`
 		HiddenLayers   int     `json:"n_layers"`
-		RopeTheta      int     `json:"rope_theta"`
+		RopeTheta      float64 `json:"rope_theta"`
 		NormEPS        float64 `json:"norm_eps"`
 	}
 
@@ -133,6 +132,7 @@ func getAltParams(dirpath string) (*Params, error) {
 	}
 
 	params := &Params{
+		Architectures:  []string{"LlamaForCausalLM"},
 		HiddenSize:     tparams.HiddenSize,
 		AttentionHeads: tparams.AttentionHeads,
 		KeyValHeads:    tparams.KeyValHeads,
@@ -229,37 +229,38 @@ func (m *TorchFormat) GetLayerName(n string) (string, error) {
 }
 
 func (r torchWriterTo) WriteTo(w io.Writer) (n int64, err error) {
-	// use the handler if one is present
-	if r.handler != nil {
-		return 0, r.handler(w, r)
+	var f32s []float32
+	switch s := r.storage.(type) {
+	case *pytorch.FloatStorage:
+		f32s = s.Data
+	case *pytorch.HalfStorage:
+		f32s = s.Data
+	case *pytorch.BFloat16Storage:
+		f32s = s.Data
+	default:
+		return 0, fmt.Errorf("unknown data type: %T", s)
 	}
 
-	switch r.storage.(type) {
-	case *pytorch.FloatStorage:
-		slog.Warn(fmt.Sprintf("unexpected storage found for layer '%s'; skipping", r.t.Name))
-		return 0, nil
-	case *pytorch.HalfStorage:
-		switch r.t.Kind {
-		case 0:
-			data := r.storage.(*pytorch.HalfStorage).Data
-			slog.Debug(fmt.Sprintf("%35s F32 (%d)", r.t.Name, len(data)))
-			if err := binary.Write(w, r.bo, data); err != nil {
-				return 0, err
-			}
-		case 1:
-			data := r.storage.(*pytorch.HalfStorage).Data
-			tData := make([]uint16, len(data))
-			for cnt, v := range data {
-				tData[cnt] = uint16(float16.Fromfloat32(v))
-			}
-			slog.Debug(fmt.Sprintf("%35s F16 (%d)", r.t.Name, len(tData)))
-			if err := binary.Write(w, r.bo, tData); err != nil {
-				return 0, err
-			}
+	if r.repacker != nil {
+		f32s, err = r.repacker(r.t.Name, f32s, r.t.Shape)
+		if err != nil {
+			return 0, err
 		}
 	}
 
-	return 0, nil
+	switch r.t.Kind {
+	case 0:
+		return 0, binary.Write(w, r.bo, f32s)
+	case 1:
+		f16s := make([]uint16, len(f32s))
+		for i := range f32s {
+			f16s[i] = float16.Fromfloat32(f32s[i]).Bits()
+		}
+
+		return 0, binary.Write(w, r.bo, f16s)
+	default:
+		return 0, fmt.Errorf("unknown storage type: %d", r.t.Kind)
+	}
 }
 
 func (m *TorchFormat) GetModelArch(name, dirPath string, params *Params) (ModelArch, error) {

docs/api.md

@@ -12,6 +12,7 @@
 - [Pull a Model](#pull-a-model)
 - [Push a Model](#push-a-model)
 - [Generate Embeddings](#generate-embeddings)
+- [List Running Models](#list-running-models)
 
 ## Conventions
 
@@ -249,7 +250,7 @@ curl http://localhost:11434/api/generate -d '{
 
 #### Request (Reproducible outputs)
 
-For reproducible outputs, set `temperature` to 0 and `seed` to a number:
+For reproducible outputs, set `seed` to a number:
 
 ##### Request
 
@@ -258,8 +259,7 @@ curl http://localhost:11434/api/generate -d '{
   "model": "mistral",
   "prompt": "Why is the sky blue?",
   "options": {
-    "seed": 123,
-    "temperature": 0
+    "seed": 123
   }
 }'
 ```
@@ -797,9 +797,9 @@ curl http://localhost:11434/api/show -d '{
 
 ```json
 {
-  "modelfile": "# Modelfile generated by \"ollama show\"\n# To build a new Modelfile based on this one, replace the FROM line with:\n# FROM llava:latest\n\nFROM /Users/matt/.ollama/models/blobs/sha256:200765e1283640ffbd013184bf496e261032fa75b99498a9613be4e94d63ad52\nTEMPLATE \"\"\"{{ .System }}\nUSER: {{ .Prompt }}\nASSSISTANT: \"\"\"\nPARAMETER num_ctx 4096\nPARAMETER stop \"\u003c/s\u003e\"\nPARAMETER stop \"USER:\"\nPARAMETER stop \"ASSSISTANT:\"",
-  "parameters": "num_ctx                        4096\nstop                           \u003c/s\u003e\nstop                           USER:\nstop                           ASSSISTANT:",
-  "template": "{{ .System }}\nUSER: {{ .Prompt }}\nASSSISTANT: ",
+  "modelfile": "# Modelfile generated by \"ollama show\"\n# To build a new Modelfile based on this one, replace the FROM line with:\n# FROM llava:latest\n\nFROM /Users/matt/.ollama/models/blobs/sha256:200765e1283640ffbd013184bf496e261032fa75b99498a9613be4e94d63ad52\nTEMPLATE \"\"\"{{ .System }}\nUSER: {{ .Prompt }}\nASSISTANT: \"\"\"\nPARAMETER num_ctx 4096\nPARAMETER stop \"\u003c/s\u003e\"\nPARAMETER stop \"USER:\"\nPARAMETER stop \"ASSISTANT:\"",
+  "parameters": "num_ctx                        4096\nstop                           \u003c/s\u003e\nstop                           USER:\nstop                           ASSISTANT:",
+  "template": "{{ .System }}\nUSER: {{ .Prompt }}\nASSISTANT: ",
   "details": {
     "format": "gguf",
     "family": "llama",
@@ -1035,3 +1035,47 @@ curl http://localhost:11434/api/embeddings -d '{
   ]
 }
 ```
+
+## List Running Models
+```shell
+GET /api/ps
+```
+
+List models that are currently loaded into memory.
+
+#### Examples
+
+### Request
+
+```shell
+curl http://localhost:11434/api/ps
+```
+
+#### Response
+
+A single JSON object will be returned.
+
+```json
+{
+  "models": [
+    {
+      "name": "mistral:latest",
+      "model": "mistral:latest",
+      "size": 5137025024,
+      "digest": "2ae6f6dd7a3dd734790bbbf58b8909a606e0e7e97e94b7604e0aa7ae4490e6d8",
+      "details": {
+        "parent_model": "",
+        "format": "gguf",
+        "family": "llama",
+        "families": [
+          "llama"
+        ],
+        "parameter_size": "7.2B",
+        "quantization_level": "Q4_0"
+      },
+      "expires_at": "2024-06-04T14:38:31.83753-07:00",
+      "size_vram": 5137025024
+    }
+  ]
+}
+```

docs/development.md

@@ -6,6 +6,8 @@ Install required tools:
 - go version 1.22 or higher
 - gcc version 11.4.0 or higher
 
+### MacOS
+
 ```bash
 brew install go cmake gcc
 ```

docs/faq.md

@@ -6,7 +6,7 @@ Ollama on macOS and Windows will automatically download updates. Click on the ta
 
 On Linux, re-run the install script:
 
-```
+```shell
 curl -fsSL https://ollama.com/install.sh | sh
 ```
 
@@ -30,7 +30,7 @@ To change this when using `ollama run`, use `/set parameter`:
 
 When using the API, specify the `num_ctx` parameter:
 
-```
+```shell
 curl http://localhost:11434/api/generate -d '{
   "model": "llama3",
   "prompt": "Why is the sky blue?",
@@ -40,6 +40,21 @@ curl http://localhost:11434/api/generate -d '{
 }'
 ```
 
+## How can I tell if my model was loaded onto the GPU?
+
+Use the `ollama ps` command to see what models are currently loaded into memory.
+
+```shell
+ollama ps
+NAME      	ID          	SIZE 	PROCESSOR	UNTIL
+llama3:70b	bcfb190ca3a7	42 GB	100% GPU 	4 minutes from now
+```
+
+The `Processor` column will show which memory the model was loaded in to:
+* `100% GPU` means the model was loaded entirely into the GPU
+* `100% CPU` means the model was loaded entirely in system memory
+* `48%/52% CPU/GPU` means the model was loaded partially onto both the GPU and into system memory
+
 ## How do I configure Ollama server?
 
 Ollama server can be configured with environment variables.
@@ -80,81 +95,19 @@ If Ollama is run as a systemd service, environment variables should be set using
 
 ### Setting environment variables on Windows
 
-On windows, Ollama inherits your user and system environment variables.
+On Windows, Ollama inherits your user and system environment variables.
 
-1. First Quit Ollama by clicking on it in the task bar
+1. First Quit Ollama by clicking on it in the task bar.
 
-2. Edit system environment variables from the control panel
+2. Start the Settings (Windows 11) or Control Panel (Windows 10) application and search for _environment variables_.
 
-3. Edit or create New variable(s) for your user account for `OLLAMA_HOST`, `OLLAMA_MODELS`, etc.
+3. Click on _Edit environment variables for your account_.
 
-4. Click OK/Apply to save
+4. Edit or create a new variable for your user account for `OLLAMA_HOST`, `OLLAMA_MODELS`, etc.
 
-5. Run `ollama` from a new terminal window
+5. Click OK/Apply to save.
 
-
-## How can I expose Ollama on my network?
-
-Ollama binds 127.0.0.1 port 11434 by default. Change the bind address with the `OLLAMA_HOST` environment variable.
-
-Refer to the section [above](#how-do-i-configure-ollama-server) for how to set environment variables on your platform.
-
-## How can I use Ollama with a proxy server?
-
-Ollama runs an HTTP server and can be exposed using a proxy server such as Nginx. To do so, configure the proxy to forward requests and optionally set required headers (if not exposing Ollama on the network). For example, with Nginx:
-
-```
-server {
-    listen 80;
-    server_name example.com;  # Replace with your domain or IP
-    location / {
-        proxy_pass http://localhost:11434;
-        proxy_set_header Host localhost:11434;
-    }
-}
-```
-
-## How can I use Ollama with ngrok?
-
-Ollama can be accessed using a range of tools for tunneling tools. For example with Ngrok:
-
-```
-ngrok http 11434 --host-header="localhost:11434"
-```
-
-## How can I use Ollama with Cloudflare Tunnel?
-
-To use Ollama with Cloudflare Tunnel, use the `--url` and `--http-host-header` flags:
-
-```
-cloudflared tunnel --url http://localhost:11434 --http-host-header="localhost:11434"
-```
-
-## How can I allow additional web origins to access Ollama?
-
-Ollama allows cross-origin requests from `127.0.0.1` and `0.0.0.0` by default. Additional origins can be configured with `OLLAMA_ORIGINS`.
-
-Refer to the section [above](#how-do-i-configure-ollama-server) for how to set environment variables on your platform.
-
-## Where are models stored?
-
-- macOS: `~/.ollama/models`
-- Linux: `/usr/share/ollama/.ollama/models`
-- Windows: `C:\Users\%username%\.ollama\models`
-
-### How do I set them to a different location?
-
-If a different directory needs to be used, set the environment variable `OLLAMA_MODELS` to the chosen directory.
-
-Refer to the section [above](#how-do-i-configure-ollama-server) for how to set environment variables on your platform.
-
-## Does Ollama send my prompts and answers back to ollama.com?
-
-No. Ollama runs locally, and conversation data does not leave your machine.
-
-## How can I use Ollama in Visual Studio Code?
-
-There is already a large collection of plugins available for VSCode as well as other editors that leverage Ollama. See the list of [extensions & plugins](https://github.com/ollama/ollama#extensions--plugins) at the bottom of the main repository readme.
+6. Start the Ollama application from the Windows Start menu.
 
 ## How do I use Ollama behind a proxy?
 
@@ -181,6 +134,69 @@ docker build -t ollama-with-ca .
 docker run -d -e HTTPS_PROXY=https://my.proxy.example.com -p 11434:11434 ollama-with-ca
 ```
 
+## Does Ollama send my prompts and answers back to ollama.com?
+
+No. Ollama runs locally, and conversation data does not leave your machine.
+
+## How can I expose Ollama on my network?
+
+Ollama binds 127.0.0.1 port 11434 by default. Change the bind address with the `OLLAMA_HOST` environment variable.
+
+Refer to the section [above](#how-do-i-configure-ollama-server) for how to set environment variables on your platform.
+
+## How can I use Ollama with a proxy server?
+
+Ollama runs an HTTP server and can be exposed using a proxy server such as Nginx. To do so, configure the proxy to forward requests and optionally set required headers (if not exposing Ollama on the network). For example, with Nginx:
+
+```
+server {
+    listen 80;
+    server_name example.com;  # Replace with your domain or IP
+    location / {
+        proxy_pass http://localhost:11434;
+        proxy_set_header Host localhost:11434;
+    }
+}
+```
+
+## How can I use Ollama with ngrok?
+
+Ollama can be accessed using a range of tools for tunneling tools. For example with Ngrok:
+
+```shell
+ngrok http 11434 --host-header="localhost:11434"
+```
+
+## How can I use Ollama with Cloudflare Tunnel?
+
+To use Ollama with Cloudflare Tunnel, use the `--url` and `--http-host-header` flags:
+
+```shell
+cloudflared tunnel --url http://localhost:11434 --http-host-header="localhost:11434"
+```
+
+## How can I allow additional web origins to access Ollama?
+
+Ollama allows cross-origin requests from `127.0.0.1` and `0.0.0.0` by default. Additional origins can be configured with `OLLAMA_ORIGINS`.
+
+Refer to the section [above](#how-do-i-configure-ollama-server) for how to set environment variables on your platform.
+
+## Where are models stored?
+
+- macOS: `~/.ollama/models`
+- Linux: `/usr/share/ollama/.ollama/models`
+- Windows: `C:\Users\%username%\.ollama\models`
+
+### How do I set them to a different location?
+
+If a different directory needs to be used, set the environment variable `OLLAMA_MODELS` to the chosen directory.
+
+Refer to the section [above](#how-do-i-configure-ollama-server) for how to set environment variables on your platform.
+
+## How can I use Ollama in Visual Studio Code?
+
+There is already a large collection of plugins available for VSCode as well as other editors that leverage Ollama. See the list of [extensions & plugins](https://github.com/ollama/ollama#extensions--plugins) at the bottom of the main repository readme.
+
 ## How do I use Ollama with GPU acceleration in Docker?
 
 The Ollama Docker container can be configured with GPU acceleration in Linux or Windows (with WSL2). This requires the [nvidia-container-toolkit](https://github.com/NVIDIA/nvidia-container-toolkit). See [ollama/ollama](https://hub.docker.com/r/ollama/ollama) for more details.
@@ -195,7 +211,7 @@ Open `Control Panel > Networking and Internet > View network status and tasks` a
 Click on `Configure` and open the `Advanced` tab. Search through each of the properties until you find `Large Send Offload Version 2 (IPv4)` and `Large Send Offload Version 2 (IPv6)`. *Disable* both of these
 properties.
 
-## How can I pre-load a model to get faster response times?
+## How can I preload a model into Ollama to get faster response times?
 
 If you are using the API you can preload a model by sending the Ollama server an empty request. This works with both the `/api/generate` and `/api/chat` API endpoints.
 
@@ -209,6 +225,11 @@ To use the chat completions endpoint, use:
 curl http://localhost:11434/api/chat -d '{"model": "mistral"}'
 ```
 
+To preload a model using the CLI, use the command:
+```shell
+ollama run llama3 ""
+```
+
 ## How do I keep a model loaded in memory or make it unload immediately?
 
 By default models are kept in memory for 5 minutes before being unloaded. This allows for quicker response times if you are making numerous requests to the LLM. You may, however, want to free up the memory before the 5 minutes have elapsed or keep the model loaded indefinitely. Use the `keep_alive` parameter with either the `/api/generate` and `/api/chat` API endpoints to control how long the model is left in memory.
@@ -233,8 +254,6 @@ Alternatively, you can change the amount of time all models are loaded into memo
 
 If you wish to override the `OLLAMA_KEEP_ALIVE` setting, use the `keep_alive` API parameter with the `/api/generate` or `/api/chat` API endpoints.
 
-## How do I manage the maximum number of requests the server can queue
+## How do I manage the maximum number of requests the Ollama server can queue?
 
-If too many requests are sent to the server, it will respond with a 503 error
-indicating the server is overloaded.  You can adjust how many requests may be
-queue by setting `OLLAMA_MAX_QUEUE`
+If too many requests are sent to the server, it will respond with a 503 error indicating the server is overloaded.  You can adjust how many requests may be queue by setting `OLLAMA_MAX_QUEUE`.

docs/import.md

@@ -1,170 +1,99 @@
-# Import a model
+# Import
 
-This guide walks through importing a GGUF, PyTorch or Safetensors model.
+GGUF models and select Safetensors models can be imported directly into Ollama.
 
-## Importing (GGUF)
+## Import GGUF
 
-### Step 1: Write a `Modelfile`
+A binary GGUF file can be imported directly into Ollama through a Modelfile.
 
-Start by creating a `Modelfile`. This file is the blueprint for your model, specifying weights, parameters, prompt templates and more.
-
-```
-FROM ./mistral-7b-v0.1.Q4_0.gguf
+```dockerfile
+FROM /path/to/file.gguf
 ```
 
-(Optional) many chat models require a prompt template in order to answer correctly. A default prompt template can be specified with the `TEMPLATE` instruction in the `Modelfile`:
+## Import Safetensors
 
-```
-FROM ./mistral-7b-v0.1.Q4_0.gguf
-TEMPLATE "[INST] {{ .Prompt }} [/INST]"
+If the model being imported is one of these architectures, it can be imported directly into Ollama through a Modelfile:
+
+ - LlamaForCausalLM
+ - MistralForCausalLM
+ - GemmaForCausalLM
+
+```dockerfile
+FROM /path/to/safetensors/directory
 ```
 
-### Step 2: Create the Ollama model
+For architectures not directly convertable by Ollama, see llama.cpp's [guide](https://github.com/ggerganov/llama.cpp/blob/master/README.md#prepare-and-quantize) on conversion. After conversion, see [Import GGUF](#import-gguf).
 
-Finally, create a model from your `Modelfile`:
+## Automatic Quantization
 
+> [!NOTE]
+> Automatic quantization requires v0.1.35 or higher.
+
+Ollama is capable of quantizing FP16 or FP32 models to any of the supported quantizations with the `-q/--quantize` flag in `ollama create`.
+
+```dockerfile
+FROM /path/to/my/gemma/f16/model
 ```
-ollama create example -f Modelfile
-```
-
-### Step 3: Run your model
-
-Next, test the model with `ollama run`:
-
-```
-ollama run example "What is your favourite condiment?"
-```
-
-## Importing (PyTorch & Safetensors)
-
-> Importing from PyTorch and Safetensors is a longer process than importing from GGUF. Improvements that make it easier are a work in progress.
-
-### Setup
-
-First, clone the `ollama/ollama` repo:
-
-```
-git clone git@github.com:ollama/ollama.git ollama
-cd ollama
-```
-
-and then fetch its `llama.cpp` submodule:
 
 ```shell
-git submodule init
-git submodule update llm/llama.cpp
+$ ollama create -q Q4_K_M mymodel
+transferring model data
+quantizing F16 model to Q4_K_M
+creating new layer sha256:735e246cc1abfd06e9cdcf95504d6789a6cd1ad7577108a70d9902fef503c1bd
+creating new layer sha256:0853f0ad24e5865173bbf9ffcc7b0f5d56b66fd690ab1009867e45e7d2c4db0f
+writing manifest
+success
 ```
 
-Next, install the Python dependencies:
+### Supported Quantizations
 
-```
-python3 -m venv llm/llama.cpp/.venv
-source llm/llama.cpp/.venv/bin/activate
-pip install -r llm/llama.cpp/requirements.txt
+<details>
+<summary>Legacy Quantization</summary>
+
+- `Q4_0`
+- `Q4_1`
+- `Q5_0`
+- `Q5_1`
+- `Q8_0`
+
+</details>
+
+<details>
+<summary>K-means Quantization</summary>`
+
+- `Q3_K_S`
+- `Q3_K_M`
+- `Q3_K_L`
+- `Q4_K_S`
+- `Q4_K_M`
+- `Q5_K_S`
+- `Q5_K_M`
+- `Q6_K`
+
+</details>
+
+> [!NOTE]
+> Activation-aware Weight Quantization (i.e. IQ) are not currently supported for automatic quantization however you can still import the quantized model into Ollama, see [Import GGUF](#import-gguf).
+
+## Template Detection
+
+> [!NOTE]
+> Template detection requires v0.1.42 or higher.
+
+Ollama uses model metadata, specifically `tokenizer.chat_template`, to automatically create a template appropriate for the model you're importing.
+
+```dockerfile
+FROM /path/to/my/gemma/model
 ```
 
-Then build the `quantize` tool:
-
-```
-make -C llm/llama.cpp quantize
+```shell
+$ ollama create mymodel
+transferring model data
+using autodetected template gemma-instruct
+creating new layer sha256:baa2a0edc27d19cc6b7537578a9a7ba1a4e3214dc185ed5ae43692b319af7b84
+creating new layer sha256:ba66c3309914dbef07e5149a648fd1877f030d337a4f240d444ea335008943cb
+writing manifest
+success
 ```
 
-### Clone the HuggingFace repository (optional)
-
-If the model is currently hosted in a HuggingFace repository, first clone that repository to download the raw model.
-
-Install [Git LFS](https://docs.github.com/en/repositories/working-with-files/managing-large-files/installing-git-large-file-storage), verify it's installed, and then clone the model's repository:
-
-```
-git lfs install
-git clone https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1 model
-```
-
-### Convert the model
-
-> Note: some model architectures require using specific convert scripts. For example, Qwen models require running `convert-hf-to-gguf.py` instead of `convert.py`
-
-```
-python llm/llama.cpp/convert.py ./model --outtype f16 --outfile converted.bin
-```
-
-### Quantize the model
-
-```
-llm/llama.cpp/quantize converted.bin quantized.bin q4_0
-```
-
-### Step 3: Write a `Modelfile`
-
-Next, create a `Modelfile` for your model:
-
-```
-FROM quantized.bin
-TEMPLATE "[INST] {{ .Prompt }} [/INST]"
-```
-
-### Step 4: Create the Ollama model
-
-Finally, create a model from your `Modelfile`:
-
-```
-ollama create example -f Modelfile
-```
-
-### Step 5: Run your model
-
-Next, test the model with `ollama run`:
-
-```
-ollama run example "What is your favourite condiment?"
-```
-
-## Publishing your model (optional – early alpha)
-
-Publishing models is in early alpha. If you'd like to publish your model to share with others, follow these steps:
-
-1. Create [an account](https://ollama.com/signup)
-2. Copy your Ollama public key:
-  - macOS: `cat ~/.ollama/id_ed25519.pub | pbcopy`
-  - Windows: `type %USERPROFILE%\.ollama\id_ed25519.pub`
-  - Linux: `cat /usr/share/ollama/.ollama/id_ed25519.pub`
-3. Add your public key to your [Ollama account](https://ollama.com/settings/keys)
-
-Next, copy your model to your username's namespace:
-
-```
-ollama cp example <your username>/example
-```
-
-> Note: model names may only contain lowercase letters, digits, and the characters `.`, `-`, and `_`.
-
-Then push the model:
-
-```
-ollama push <your username>/example
-```
-
-After publishing, your model will be available at `https://ollama.com/<your username>/example`.
-
-## Quantization reference
-
-The quantization options are as follow (from highest highest to lowest levels of quantization). Note: some architectures such as Falcon do not support K quants.
-
-- `q2_K`
-- `q3_K`
-- `q3_K_S`
-- `q3_K_M`
-- `q3_K_L`
-- `q4_0` (recommended)
-- `q4_1`
-- `q4_K`
-- `q4_K_S`
-- `q4_K_M`
-- `q5_0`
-- `q5_1`
-- `q5_K`
-- `q5_K_S`
-- `q5_K_M`
-- `q6_K`
-- `q8_0`
-- `f16`
+Defining a template in the Modelfile will disable this feature which may be useful if you want to use a different template than the autodetected one.

docs/linux.md

@@ -100,6 +100,16 @@ sudo curl -L https://ollama.com/download/ollama-linux-amd64 -o /usr/bin/ollama
 sudo chmod +x /usr/bin/ollama
 ```
 
+## Installing specific versions
+
+Use `OLLAMA_VERSION` environment variable with the install script to install a specific version of Ollama, including pre-releases. You can find the version numbers in the [releases page](https://github.com/ollama/ollama/releases). 
+
+For example:
+
+```
+curl -fsSL https://ollama.com/install.sh | OLLAMA_VERSION=0.1.32 sh
+```
+
 ## Viewing logs
 
 To view logs of Ollama running as a startup service, run:

docs/troubleshooting.md

@@ -1,104 +1,87 @@
-# How to troubleshoot issues
-
-Sometimes Ollama may not perform as expected. One of the best ways to figure out what happened is to take a look at the logs. Find the logs on **Mac** by running the command:
-
-```shell
-cat ~/.ollama/logs/server.log
-```
-
-On **Linux** systems with systemd, the logs can be found with this command:
-
-```shell
-journalctl -u ollama
-```
-
-When you run Ollama in a **container**, the logs go to stdout/stderr in the container:
-
-```shell
-docker logs <container-name>
-```
-(Use `docker ps` to find the container name)
-
-If manually running `ollama serve` in a terminal, the logs will be on that terminal.
-
-When you run Ollama on **Windows**, there are a few different locations.  You can view them in the explorer window by hitting `<cmd>+R` and type in:
-- `explorer %LOCALAPPDATA%\Ollama` to view logs
-- `explorer %LOCALAPPDATA%\Programs\Ollama` to browse the binaries (The installer adds this to your user PATH)
-- `explorer %HOMEPATH%\.ollama` to browse where models and configuration is stored
-- `explorer %TEMP%` where temporary executable files are stored in one or more `ollama*` directories
-
-To enable additional debug logging to help troubleshoot problems, first **Quit the running app from the tray menu** then in a powershell terminal
-```powershell
-$env:OLLAMA_DEBUG="1"
-& "ollama app.exe"
-```
-
-Join the [Discord](https://discord.gg/ollama) for help interpreting the logs.
-
-## LLM libraries
-
-Ollama includes multiple LLM libraries compiled for different GPUs and CPU
-vector features.  Ollama tries to pick the best one based on the capabilities of
-your system.  If this autodetection has problems, or you run into other problems
-(e.g. crashes in your GPU) you can workaround this by forcing a specific LLM
-library.  `cpu_avx2` will perform the best, followed by `cpu_avx` an the slowest
-but most compatible is `cpu`.  Rosetta emulation under MacOS will work with the
-`cpu` library. 
-
-In the server log, you will see a message that looks something like this (varies
-from release to release):
-
-```
-Dynamic LLM libraries [rocm_v6 cpu cpu_avx cpu_avx2 cuda_v11 rocm_v5]
-```
-
-**Experimental LLM Library Override**
-
-You can set OLLAMA_LLM_LIBRARY to any of the available LLM libraries to bypass
-autodetection, so for example, if you have a CUDA card, but want to force the
-CPU LLM library with AVX2 vector support, use:
-
-```
-OLLAMA_LLM_LIBRARY="cpu_avx2" ollama serve
-```
-
-You can see what features your CPU has with the following.  
-```
-cat /proc/cpuinfo| grep flags  | head -1
-```
-
-## Installing older or pre-release versions on Linux
-
-If you run into problems on Linux and want to install an older version, or you'd
-like to try out a pre-release before it's officially released, you can tell the
-install script which version to install.
-
-```sh
-curl -fsSL https://ollama.com/install.sh | OLLAMA_VERSION="0.1.29" sh
-```
-
-## Linux tmp noexec 
-
-If your system is configured with the "noexec" flag where Ollama stores its
-temporary executable files, you can specify an alternate location by setting
-OLLAMA_TMPDIR to a location writable by the user ollama runs as.  For example
-OLLAMA_TMPDIR=/usr/share/ollama/
-
-## Container fails to run on NVIDIA GPU
-
-Make sure you've set up the conatiner runtime first as described in [docker.md](./docker.md)
-
-Sometimes the container runtime can have difficulties initializing the GPU.
-When you check the server logs, this can show up as various error codes, such
-as "3" (not initialized), "46" (device unavailable), "100" (no device), "999"
-(unknown), or others.  The following troubleshooting techniques may help resolve
-the problem
-
-- Is the uvm driver not loaded? `sudo nvidia-modprobe -u`
-- Try reloading the nvidia_uvm driver - `sudo rmmod nvidia_uvm` then `sudo modprobe nvidia_uvm`
-- Try rebooting
-- Make sure you're running the latest nvidia drivers
-
-If none of those resolve the problem, gather additional information and file an issue:
-- Set `CUDA_ERROR_LEVEL=50` and try again to get more diagnostic logs
-- Check dmesg for any errors `sudo dmesg | grep -i nvrm` and `sudo dmesg | grep -i nvidia`
+# How to troubleshoot issues
+
+Sometimes Ollama may not perform as expected. One of the best ways to figure out what happened is to take a look at the logs. Find the logs on **Mac** by running the command:
+
+```shell
+cat ~/.ollama/logs/server.log
+```
+
+On **Linux** systems with systemd, the logs can be found with this command:
+
+```shell
+journalctl -u ollama
+```
+
+When you run Ollama in a **container**, the logs go to stdout/stderr in the container:
+
+```shell
+docker logs <container-name>
+```
+(Use `docker ps` to find the container name)
+
+If manually running `ollama serve` in a terminal, the logs will be on that terminal.
+
+When you run Ollama on **Windows**, there are a few different locations. You can view them in the explorer window by hitting `<cmd>+R` and type in:
+- `explorer %LOCALAPPDATA%\Ollama` to view logs
+- `explorer %LOCALAPPDATA%\Programs\Ollama` to browse the binaries (The installer adds this to your user PATH)
+- `explorer %HOMEPATH%\.ollama` to browse where models and configuration is stored
+- `explorer %TEMP%` where temporary executable files are stored in one or more `ollama*` directories
+
+To enable additional debug logging to help troubleshoot problems, first **Quit the running app from the tray menu** then in a powershell terminal
+```powershell
+$env:OLLAMA_DEBUG="1"
+& "ollama app.exe"
+```
+
+Join the [Discord](https://discord.gg/ollama) for help interpreting the logs.
+
+## LLM libraries
+
+Ollama includes multiple LLM libraries compiled for different GPUs and CPU vector features. Ollama tries to pick the best one based on the capabilities of your system. If this autodetection has problems, or you run into other problems (e.g. crashes in your GPU) you can workaround this by forcing a specific LLM library. `cpu_avx2` will perform the best, followed by `cpu_avx` an the slowest but most compatible is `cpu`. Rosetta emulation under MacOS will work with the `cpu` library. 
+
+In the server log, you will see a message that looks something like this (varies from release to release):
+
+```
+Dynamic LLM libraries [rocm_v6 cpu cpu_avx cpu_avx2 cuda_v11 rocm_v5]
+```
+
+**Experimental LLM Library Override**
+
+You can set OLLAMA_LLM_LIBRARY to any of the available LLM libraries to bypass autodetection, so for example, if you have a CUDA card, but want to force the CPU LLM library with AVX2 vector support, use:
+
+```
+OLLAMA_LLM_LIBRARY="cpu_avx2" ollama serve
+```
+
+You can see what features your CPU has with the following.
+```
+cat /proc/cpuinfo| grep flags | head -1
+```
+
+## Installing older or pre-release versions on Linux
+
+If you run into problems on Linux and want to install an older version, or you'd like to try out a pre-release before it's officially released, you can tell the install script which version to install.
+
+```sh
+curl -fsSL https://ollama.com/install.sh | OLLAMA_VERSION="0.1.29" sh
+```
+
+## Linux tmp noexec 
+
+If your system is configured with the "noexec" flag where Ollama stores its temporary executable files, you can specify an alternate location by setting OLLAMA_TMPDIR to a location writable by the user ollama runs as. For example OLLAMA_TMPDIR=/usr/share/ollama/
+
+## Container fails to run on NVIDIA GPU
+
+Make sure you've set up the container runtime first as described in [docker.md](./docker.md)
+
+Sometimes the container runtime can have difficulties initializing the GPU. When you check the server logs, this can show up as various error codes, such as "3" (not initialized), "46" (device unavailable), "100" (no device), "999" (unknown), or others. The following troubleshooting techniques may help resolve the problem
+
+- Is the container runtime working?  Try `docker run --gpus all ubuntu nvidia-smi` - if this doesn't work, Ollama wont be able to see your NVIDIA GPU.
+- Is the uvm driver not loaded? `sudo nvidia-modprobe -u`
+- Try reloading the nvidia_uvm driver - `sudo rmmod nvidia_uvm` then `sudo modprobe nvidia_uvm`
+- Try rebooting
+- Make sure you're running the latest nvidia drivers
+
+If none of those resolve the problem, gather additional information and file an issue:
+- Set `CUDA_ERROR_LEVEL=50` and try again to get more diagnostic logs
+- Check dmesg for any errors `sudo dmesg | grep -i nvrm` and `sudo dmesg | grep -i nvidia`

docs/tutorials/langchainpy.md

@@ -45,7 +45,7 @@ all_splits = text_splitter.split_documents(data)
 ```
 
 It's split up, but we have to find the relevant splits and then submit those to the model. We can do this by creating embeddings and storing them in a vector database. We can use Ollama directly to instantiate an embedding model. We will use ChromaDB in this example for a vector database. `pip install chromadb`
-
+We also need to pull embedding model: `ollama pull nomic-embed-text`
 ```python
 from langchain.embeddings import OllamaEmbeddings
 from langchain.vectorstores import Chroma
@@ -68,7 +68,8 @@ The next thing is to send the question and the relevant parts of the docs to the
 ```python
 from langchain.chains import RetrievalQA
 qachain=RetrievalQA.from_chain_type(ollama, retriever=vectorstore.as_retriever())
-qachain.invoke({"query": question})
+res = qachain.invoke({"query": question})
+print(res['result'])
 ```
 
 The answer received from this chain was:

docs/windows.md

@@ -33,7 +33,7 @@ Here's a quick example showing API access from `powershell`
 ## Troubleshooting
 
 While we're in preview, `OLLAMA_DEBUG` is always enabled, which adds
-a "view logs" menu item to the app, and increses logging for the GUI app and
+a "view logs" menu item to the app, and increases logging for the GUI app and
 server.
 
 Ollama on Windows stores files in a few different locations.  You can view them in

envconfig/config.go

@@ -0,0 +1,302 @@
+package envconfig
+
+import (
+	"errors"
+	"fmt"
+	"log/slog"
+	"net"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+)
+
+type OllamaHost struct {
+	Scheme string
+	Host   string
+	Port   string
+}
+
+func (o OllamaHost) String() string {
+	return fmt.Sprintf("%s://%s:%s", o.Scheme, o.Host, o.Port)
+}
+
+var ErrInvalidHostPort = errors.New("invalid port specified in OLLAMA_HOST")
+
+var (
+	// Set via OLLAMA_ORIGINS in the environment
+	AllowOrigins []string
+	// Set via OLLAMA_DEBUG in the environment
+	Debug bool
+	// Experimental flash attention
+	FlashAttention bool
+	// Set via OLLAMA_HOST in the environment
+	Host *OllamaHost
+	// Set via OLLAMA_KEEP_ALIVE in the environment
+	KeepAlive string
+	// Set via OLLAMA_LLM_LIBRARY in the environment
+	LLMLibrary string
+	// Set via OLLAMA_MAX_LOADED_MODELS in the environment
+	MaxRunners int
+	// Set via OLLAMA_MAX_QUEUE in the environment
+	MaxQueuedRequests int
+	// Set via OLLAMA_MODELS in the environment
+	ModelsDir string
+	// Set via OLLAMA_MAX_VRAM in the environment
+	MaxVRAM uint64
+	// Set via OLLAMA_NOHISTORY in the environment
+	NoHistory bool
+	// Set via OLLAMA_NOPRUNE in the environment
+	NoPrune bool
+	// Set via OLLAMA_NUM_PARALLEL in the environment
+	NumParallel int
+	// Set via OLLAMA_RUNNERS_DIR in the environment
+	RunnersDir string
+	// Set via OLLAMA_TMPDIR in the environment
+	TmpDir string
+)
+
+type EnvVar struct {
+	Name        string
+	Value       any
+	Description string
+}
+
+func AsMap() map[string]EnvVar {
+	return map[string]EnvVar{
+		"OLLAMA_DEBUG":             {"OLLAMA_DEBUG", Debug, "Show additional debug information (e.g. OLLAMA_DEBUG=1)"},
+		"OLLAMA_FLASH_ATTENTION":   {"OLLAMA_FLASH_ATTENTION", FlashAttention, "Enabled flash attention"},
+		"OLLAMA_HOST":              {"OLLAMA_HOST", Host, "IP Address for the ollama server (default 127.0.0.1:11434)"},
+		"OLLAMA_KEEP_ALIVE":        {"OLLAMA_KEEP_ALIVE", KeepAlive, "The duration that models stay loaded in memory (default \"5m\")"},
+		"OLLAMA_LLM_LIBRARY":       {"OLLAMA_LLM_LIBRARY", LLMLibrary, "Set LLM library to bypass autodetection"},
+		"OLLAMA_MAX_LOADED_MODELS": {"OLLAMA_MAX_LOADED_MODELS", MaxRunners, "Maximum number of loaded models (default 1)"},
+		"OLLAMA_MAX_QUEUE":         {"OLLAMA_MAX_QUEUE", MaxQueuedRequests, "Maximum number of queued requests"},
+		"OLLAMA_MAX_VRAM":          {"OLLAMA_MAX_VRAM", MaxVRAM, "Maximum VRAM"},
+		"OLLAMA_MODELS":            {"OLLAMA_MODELS", ModelsDir, "The path to the models directory"},
+		"OLLAMA_NOHISTORY":         {"OLLAMA_NOHISTORY", NoHistory, "Do not preserve readline history"},
+		"OLLAMA_NOPRUNE":           {"OLLAMA_NOPRUNE", NoPrune, "Do not prune model blobs on startup"},
+		"OLLAMA_NUM_PARALLEL":      {"OLLAMA_NUM_PARALLEL", NumParallel, "Maximum number of parallel requests (default 1)"},
+		"OLLAMA_ORIGINS":           {"OLLAMA_ORIGINS", AllowOrigins, "A comma separated list of allowed origins"},
+		"OLLAMA_RUNNERS_DIR":       {"OLLAMA_RUNNERS_DIR", RunnersDir, "Location for runners"},
+		"OLLAMA_TMPDIR":            {"OLLAMA_TMPDIR", TmpDir, "Location for temporary files"},
+	}
+}
+
+func Values() map[string]string {
+	vals := make(map[string]string)
+	for k, v := range AsMap() {
+		vals[k] = fmt.Sprintf("%v", v.Value)
+	}
+	return vals
+}
+
+var defaultAllowOrigins = []string{
+	"localhost",
+	"127.0.0.1",
+	"0.0.0.0",
+}
+
+// Clean quotes and spaces from the value
+func clean(key string) string {
+	return strings.Trim(os.Getenv(key), "\"' ")
+}
+
+func init() {
+	// default values
+	NumParallel = 1
+	MaxRunners = 1
+	MaxQueuedRequests = 512
+
+	LoadConfig()
+}
+
+func LoadConfig() {
+	if debug := clean("OLLAMA_DEBUG"); debug != "" {
+		d, err := strconv.ParseBool(debug)
+		if err == nil {
+			Debug = d
+		} else {
+			Debug = true
+		}
+	}
+
+	if fa := clean("OLLAMA_FLASH_ATTENTION"); fa != "" {
+		d, err := strconv.ParseBool(fa)
+		if err == nil {
+			FlashAttention = d
+		}
+	}
+
+	RunnersDir = clean("OLLAMA_RUNNERS_DIR")
+	if runtime.GOOS == "windows" && RunnersDir == "" {
+		// On Windows we do not carry the payloads inside the main executable
+		appExe, err := os.Executable()
+		if err != nil {
+			slog.Error("failed to lookup executable path", "error", err)
+		}
+
+		cwd, err := os.Getwd()
+		if err != nil {
+			slog.Error("failed to lookup working directory", "error", err)
+		}
+
+		var paths []string
+		for _, root := range []string{filepath.Dir(appExe), cwd} {
+			paths = append(paths,
+				root,
+				filepath.Join(root, "windows-"+runtime.GOARCH),
+				filepath.Join(root, "dist", "windows-"+runtime.GOARCH),
+			)
+		}
+
+		// Try a few variations to improve developer experience when building from source in the local tree
+		for _, p := range paths {
+			candidate := filepath.Join(p, "ollama_runners")
+			_, err := os.Stat(candidate)
+			if err == nil {
+				RunnersDir = candidate
+				break
+			}
+		}
+		if RunnersDir == "" {
+			slog.Error("unable to locate llm runner directory.  Set OLLAMA_RUNNERS_DIR to the location of 'ollama_runners'")
+		}
+	}
+
+	TmpDir = clean("OLLAMA_TMPDIR")
+
+	userLimit := clean("OLLAMA_MAX_VRAM")
+	if userLimit != "" {
+		avail, err := strconv.ParseUint(userLimit, 10, 64)
+		if err != nil {
+			slog.Error("invalid setting, ignoring", "OLLAMA_MAX_VRAM", userLimit, "error", err)
+		} else {
+			MaxVRAM = avail
+		}
+	}
+
+	LLMLibrary = clean("OLLAMA_LLM_LIBRARY")
+
+	if onp := clean("OLLAMA_NUM_PARALLEL"); onp != "" {
+		val, err := strconv.Atoi(onp)
+		if err != nil || val <= 0 {
+			slog.Error("invalid setting must be greater than zero", "OLLAMA_NUM_PARALLEL", onp, "error", err)
+		} else {
+			NumParallel = val
+		}
+	}
+
+	if nohistory := clean("OLLAMA_NOHISTORY"); nohistory != "" {
+		NoHistory = true
+	}
+
+	if noprune := clean("OLLAMA_NOPRUNE"); noprune != "" {
+		NoPrune = true
+	}
+
+	if origins := clean("OLLAMA_ORIGINS"); origins != "" {
+		AllowOrigins = strings.Split(origins, ",")
+	}
+	for _, allowOrigin := range defaultAllowOrigins {
+		AllowOrigins = append(AllowOrigins,
+			fmt.Sprintf("http://%s", allowOrigin),
+			fmt.Sprintf("https://%s", allowOrigin),
+			fmt.Sprintf("http://%s", net.JoinHostPort(allowOrigin, "*")),
+			fmt.Sprintf("https://%s", net.JoinHostPort(allowOrigin, "*")),
+		)
+	}
+
+	AllowOrigins = append(AllowOrigins,
+		"app://*",
+		"file://*",
+		"tauri://*",
+	)
+
+	maxRunners := clean("OLLAMA_MAX_LOADED_MODELS")
+	if maxRunners != "" {
+		m, err := strconv.Atoi(maxRunners)
+		if err != nil {
+			slog.Error("invalid setting", "OLLAMA_MAX_LOADED_MODELS", maxRunners, "error", err)
+		} else {
+			MaxRunners = m
+		}
+	}
+
+	if onp := os.Getenv("OLLAMA_MAX_QUEUE"); onp != "" {
+		p, err := strconv.Atoi(onp)
+		if err != nil || p <= 0 {
+			slog.Error("invalid setting", "OLLAMA_MAX_QUEUE", onp, "error", err)
+		} else {
+			MaxQueuedRequests = p
+		}
+	}
+
+	KeepAlive = clean("OLLAMA_KEEP_ALIVE")
+
+	var err error
+	ModelsDir, err = getModelsDir()
+	if err != nil {
+		slog.Error("invalid setting", "OLLAMA_MODELS", ModelsDir, "error", err)
+	}
+
+	Host, err = getOllamaHost()
+	if err != nil {
+		slog.Error("invalid setting", "OLLAMA_HOST", Host, "error", err, "using default port", Host.Port)
+	}
+}
+
+func getModelsDir() (string, error) {
+	if models, exists := os.LookupEnv("OLLAMA_MODELS"); exists {
+		return models, nil
+	}
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(home, ".ollama", "models"), nil
+}
+
+func getOllamaHost() (*OllamaHost, error) {
+	defaultPort := "11434"
+
+	hostVar := os.Getenv("OLLAMA_HOST")
+	hostVar = strings.TrimSpace(strings.Trim(strings.TrimSpace(hostVar), "\"'"))
+
+	scheme, hostport, ok := strings.Cut(hostVar, "://")
+	switch {
+	case !ok:
+		scheme, hostport = "http", hostVar
+	case scheme == "http":
+		defaultPort = "80"
+	case scheme == "https":
+		defaultPort = "443"
+	}
+
+	// trim trailing slashes
+	hostport = strings.TrimRight(hostport, "/")
+
+	host, port, err := net.SplitHostPort(hostport)
+	if err != nil {
+		host, port = "127.0.0.1", defaultPort
+		if ip := net.ParseIP(strings.Trim(hostport, "[]")); ip != nil {
+			host = ip.String()
+		} else if hostport != "" {
+			host = hostport
+		}
+	}
+
+	if portNum, err := strconv.ParseInt(port, 10, 32); err != nil || portNum > 65535 || portNum < 0 {
+		return &OllamaHost{
+			Scheme: scheme,
+			Host:   host,
+			Port:   defaultPort,
+		}, ErrInvalidHostPort
+	}
+
+	return &OllamaHost{
+		Scheme: scheme,
+		Host:   host,
+		Port:   port,
+	}, nil
+}

envconfig/config_test.go

@@ -0,0 +1,71 @@
+package envconfig
+
+import (
+	"fmt"
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConfig(t *testing.T) {
+	Debug = false // Reset whatever was loaded in init()
+	t.Setenv("OLLAMA_DEBUG", "")
+	LoadConfig()
+	require.False(t, Debug)
+	t.Setenv("OLLAMA_DEBUG", "false")
+	LoadConfig()
+	require.False(t, Debug)
+	t.Setenv("OLLAMA_DEBUG", "1")
+	LoadConfig()
+	require.True(t, Debug)
+	t.Setenv("OLLAMA_FLASH_ATTENTION", "1")
+	LoadConfig()
+	require.True(t, FlashAttention)
+}
+
+func TestClientFromEnvironment(t *testing.T) {
+	type testCase struct {
+		value  string
+		expect string
+		err    error
+	}
+
+	hostTestCases := map[string]*testCase{
+		"empty":               {value: "", expect: "127.0.0.1:11434"},
+		"only address":        {value: "1.2.3.4", expect: "1.2.3.4:11434"},
+		"only port":           {value: ":1234", expect: ":1234"},
+		"address and port":    {value: "1.2.3.4:1234", expect: "1.2.3.4:1234"},
+		"hostname":            {value: "example.com", expect: "example.com:11434"},
+		"hostname and port":   {value: "example.com:1234", expect: "example.com:1234"},
+		"zero port":           {value: ":0", expect: ":0"},
+		"too large port":      {value: ":66000", err: ErrInvalidHostPort},
+		"too small port":      {value: ":-1", err: ErrInvalidHostPort},
+		"ipv6 localhost":      {value: "[::1]", expect: "[::1]:11434"},
+		"ipv6 world open":     {value: "[::]", expect: "[::]:11434"},
+		"ipv6 no brackets":    {value: "::1", expect: "[::1]:11434"},
+		"ipv6 + port":         {value: "[::1]:1337", expect: "[::1]:1337"},
+		"extra space":         {value: " 1.2.3.4 ", expect: "1.2.3.4:11434"},
+		"extra quotes":        {value: "\"1.2.3.4\"", expect: "1.2.3.4:11434"},
+		"extra space+quotes":  {value: " \" 1.2.3.4 \" ", expect: "1.2.3.4:11434"},
+		"extra single quotes": {value: "'1.2.3.4'", expect: "1.2.3.4:11434"},
+	}
+
+	for k, v := range hostTestCases {
+		t.Run(k, func(t *testing.T) {
+			t.Setenv("OLLAMA_HOST", v.value)
+			LoadConfig()
+
+			oh, err := getOllamaHost()
+			if err != v.err {
+				t.Fatalf("expected %s, got %s", v.err, err)
+			}
+
+			if err == nil {
+				host := net.JoinHostPort(oh.Host, oh.Port)
+				assert.Equal(t, v.expect, host, fmt.Sprintf("%s: expected %s, got %s", k, v.expect, host))
+			}
+		})
+	}
+}

examples/langchain-python-rag-privategpt/ingest.py

@@ -77,13 +77,21 @@ LOADER_MAPPING = {
 
 
 def load_single_document(file_path: str) -> List[Document]:
-    ext = "." + file_path.rsplit(".", 1)[-1]
-    if ext in LOADER_MAPPING:
-        loader_class, loader_args = LOADER_MAPPING[ext]
-        loader = loader_class(file_path, **loader_args)
-        return loader.load()
+    if os.path.getsize(file_path) != 0:
+        filename, ext = os.path.splitext(file_path)
+        if ext in LOADER_MAPPING:
+            loader_class, loader_args = LOADER_MAPPING[ext]
+            try:
+                loader = loader_class(file_path, **loader_args)
+                if loader:
+                    return loader.load()
+            except:
+                print(f"Corrupted file {file_path}. Ignoring it.")
+        else:
+            print(f"Unsupported file {file_path}. Ignoring it.")
+    else:
+        print(f"Empty file {file_path}. Ignoring it.")
 
-    raise ValueError(f"Unsupported file extension '{ext}'")
 
 def load_documents(source_dir: str, ignored_files: List[str] = []) -> List[Document]:
     """
@@ -100,7 +108,8 @@ def load_documents(source_dir: str, ignored_files: List[str] = []) -> List[Docum
         results = []
         with tqdm(total=len(filtered_files), desc='Loading new documents', ncols=80) as pbar:
             for i, docs in enumerate(pool.imap_unordered(load_single_document, filtered_files)):
-                results.extend(docs)
+                if docs:
+                    results.extend(docs)
                 pbar.update()
 
     return results

examples/langchain-python-rag-privategpt/requirements.txt

@@ -11,4 +11,5 @@ tabulate==0.9.0
 pandoc==2.3
 pypandoc==1.11
 tqdm==4.66.1
-sentence_transformers==2.2.2
+sentence_transformers==2.2.2
+numpy>=1.22.2 # not directly required, pinned by Snyk to avoid a vulnerability

examples/python-simplechat/client.py

@@ -9,6 +9,7 @@ def chat(messages):
     r = requests.post(
         "http://0.0.0.0:11434/api/chat",
         json={"model": model, "messages": messages, "stream": True},
+	stream=True
     )
     r.raise_for_status()
     output = ""

format/format_test.go

@@ -5,7 +5,6 @@ import (
 )
 
 func TestHumanNumber(t *testing.T) {
-
 	type testCase struct {
 		input    uint64
 		expected string

format/time.go

@@ -60,7 +60,9 @@ func humanTime(t time.Time, zeroValue string) string {
 	}
 
 	delta := time.Since(t)
-	if delta < 0 {
+	if int(delta.Hours())/24/365 < -20 {
+		return "Forever"
+	} else if delta < 0 {
 		return humanDuration(-delta) + " from now"
 	}
 

format/time_test.go

@@ -32,4 +32,14 @@ func TestHumanTime(t *testing.T) {
 		v := now.Add(800 * time.Millisecond)
 		assertEqual(t, HumanTime(v, ""), "Less than a second from now")
 	})
+
+	t.Run("time way in the future", func(t *testing.T) {
+		v := now.Add(24 * time.Hour * 365 * 200)
+		assertEqual(t, HumanTime(v, ""), "Forever")
+	})
+
+	t.Run("time way in the future lowercase", func(t *testing.T) {
+		v := now.Add(24 * time.Hour * 365 * 200)
+		assertEqual(t, HumanTimeLower(v, ""), "forever")
+	})
 }

go.mod

@@ -4,12 +4,10 @@ go 1.22.0
 
 require (
 	github.com/containerd/console v1.0.3
-	github.com/d4l3k/go-bfloat16 v0.0.0-20211005043715-690c3bdd05f1
 	github.com/emirpasic/gods v1.18.1
 	github.com/gin-gonic/gin v1.10.0
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/uuid v1.1.2
-	github.com/mitchellh/mapstructure v1.5.0
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/spf13/cobra v1.7.0
 	github.com/stretchr/testify v1.9.0
@@ -18,6 +16,9 @@ require (
 )
 
 require (
+	github.com/agnivade/levenshtein v1.1.1
+	github.com/d4l3k/go-bfloat16 v0.0.0-20211005043715-690c3bdd05f1
+	github.com/mattn/go-runewidth v0.0.14
 	github.com/nlpodyssey/gopickle v0.3.0
 	github.com/pdevine/tensor v0.0.0-20240510204454-f88f4562727c
 )
@@ -33,7 +34,6 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/google/flatbuffers v24.3.25+incompatible // indirect
 	github.com/kr/text v0.2.0 // indirect
-	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect

go.sum

@@ -4,10 +4,14 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7
 gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8=
+github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 h1:q4dksr6ICHXqG5hm0ZW5IHyeEJXoIJSOZeBLmWPNeIQ=
 github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40/go.mod h1:Q7yQnSMnLvcXlZ8RV+jwz/6y1rQTqbX6C82SndT52Zs=
+github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
+github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
 github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
 github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
 github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
@@ -36,6 +40,8 @@ github.com/d4l3k/go-bfloat16 v0.0.0-20211005043715-690c3bdd05f1/go.mod h1:uw2gLc
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g=
+github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
 github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -135,8 +141,6 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
 github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
-github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=

gpu/amd_windows.go

@@ -65,7 +65,7 @@ func AMDGetGPUInfo() []GpuInfo {
 
 	slog.Debug("detected hip devices", "count", count)
 	// TODO how to determine the underlying device ID when visible devices is causing this to subset?
-	for i := 0; i < count; i++ {
+	for i := range count {
 		err = hl.HipSetDevice(i)
 		if err != nil {
 			slog.Warn("set device", "id", i, "error", err)

gpu/assets.go

@@ -13,7 +13,7 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/ollama/ollama/server/envconfig"
+	"github.com/ollama/ollama/envconfig"
 )
 
 var (
@@ -80,7 +80,7 @@ func cleanupTmpDirs() {
 		if err == nil {
 			pid, err := strconv.Atoi(string(raw))
 			if err == nil {
-				if proc, err := os.FindProcess(int(pid)); err == nil && !errors.Is(proc.Signal(syscall.Signal(0)), os.ErrProcessDone) {
+				if proc, err := os.FindProcess(pid); err == nil && !errors.Is(proc.Signal(syscall.Signal(0)), os.ErrProcessDone) {
 					// Another running ollama, ignore this tmpdir
 					continue
 				}

gpu/cuda_common.go

@@ -18,5 +18,4 @@ func cudaGetVisibleDevicesEnv(gpuInfo []GpuInfo) (string, string) {
 		ids = append(ids, info.ID)
 	}
 	return "CUDA_VISIBLE_DEVICES", strings.Join(ids, ",")
-
 }

gpu/gpu.go

@@ -20,14 +20,15 @@ import (
 	"sync"
 	"unsafe"
 
+	"github.com/ollama/ollama/envconfig"
 	"github.com/ollama/ollama/format"
-	"github.com/ollama/ollama/server/envconfig"
 )
 
 type handles struct {
 	deviceCount int
 	cudart      *C.cudart_handle_t
 	nvcuda      *C.nvcuda_handle_t
+	oneapi      *C.oneapi_handle_t
 }
 
 const (
@@ -80,6 +81,15 @@ var NvcudaWindowsGlobs = []string{
 	"c:\\windows\\system*\\nvcuda.dll",
 }
 
+var OneapiWindowsGlobs = []string{
+	"c:\\Windows\\System32\\DriverStore\\FileRepository\\*\\ze_intel_gpu64.dll",
+}
+
+var OneapiLinuxGlobs = []string{
+	"/usr/lib/x86_64-linux-gnu/libze_intel_gpu.so*",
+	"/usr/lib*/libze_intel_gpu.so*",
+}
+
 // Jetson devices have JETSON_JETPACK="x.y.z" factory set to the Jetpack version installed.
 // Included to drive logic for reducing Ollama-allocated overhead on L4T/Jetson devices.
 var CudaTegra string = os.Getenv("JETSON_JETPACK")
@@ -141,6 +151,7 @@ func initGPUHandles() *handles {
 			return gpuHandles
 		}
 	}
+
 	return gpuHandles
 }
 
@@ -176,44 +187,46 @@ func GetGPUInfo() GpuInfoList {
 	resp := []GpuInfo{}
 
 	// NVIDIA first
-	for i := 0; i < gpuHandles.deviceCount; i++ {
+	for i := range gpuHandles.deviceCount {
 		// TODO once we support CPU compilation variants of GPU libraries refine this...
 		if cpuVariant == "" && runtime.GOARCH == "amd64" {
 			continue
 		}
-		gpuInfo := GpuInfo{
-			Library: "cuda",
-		}
-		var driverMajor int
-		var driverMinor int
-		if gpuHandles.cudart != nil {
-			C.cudart_check_vram(*gpuHandles.cudart, C.int(i), &memInfo)
-		} else {
-			C.nvcuda_check_vram(*gpuHandles.nvcuda, C.int(i), &memInfo)
-			driverMajor = int(gpuHandles.nvcuda.driver_major)
-			driverMinor = int(gpuHandles.nvcuda.driver_minor)
-		}
-		if memInfo.err != nil {
-			slog.Info("error looking up nvidia GPU memory", "error", C.GoString(memInfo.err))
-			C.free(unsafe.Pointer(memInfo.err))
-			continue
-		}
-		if memInfo.major < CudaComputeMin[0] || (memInfo.major == CudaComputeMin[0] && memInfo.minor < CudaComputeMin[1]) {
-			slog.Info(fmt.Sprintf("[%d] CUDA GPU is too old. Compute Capability detected: %d.%d", i, memInfo.major, memInfo.minor))
-			continue
-		}
-		gpuInfo.TotalMemory = uint64(memInfo.total)
-		gpuInfo.FreeMemory = uint64(memInfo.free)
-		gpuInfo.ID = C.GoString(&memInfo.gpu_id[0])
-		gpuInfo.Compute = fmt.Sprintf("%d.%d", memInfo.major, memInfo.minor)
-		gpuInfo.MinimumMemory = cudaMinimumMemory
-		gpuInfo.DependencyPath = depPath
-		gpuInfo.Name = C.GoString(&memInfo.gpu_name[0])
-		gpuInfo.DriverMajor = int(driverMajor)
-		gpuInfo.DriverMinor = int(driverMinor)
+		if gpuHandles.cudart != nil || gpuHandles.nvcuda != nil {
+			gpuInfo := GpuInfo{
+				Library: "cuda",
+			}
+			var driverMajor int
+			var driverMinor int
+			if gpuHandles.cudart != nil {
+				C.cudart_check_vram(*gpuHandles.cudart, C.int(i), &memInfo)
+			} else {
+				C.nvcuda_check_vram(*gpuHandles.nvcuda, C.int(i), &memInfo)
+				driverMajor = int(gpuHandles.nvcuda.driver_major)
+				driverMinor = int(gpuHandles.nvcuda.driver_minor)
+			}
+			if memInfo.err != nil {
+				slog.Info("error looking up nvidia GPU memory", "error", C.GoString(memInfo.err))
+				C.free(unsafe.Pointer(memInfo.err))
+				continue
+			}
+			if memInfo.major < CudaComputeMin[0] || (memInfo.major == CudaComputeMin[0] && memInfo.minor < CudaComputeMin[1]) {
+				slog.Info(fmt.Sprintf("[%d] CUDA GPU is too old. Compute Capability detected: %d.%d", i, memInfo.major, memInfo.minor))
+				continue
+			}
+			gpuInfo.TotalMemory = uint64(memInfo.total)
+			gpuInfo.FreeMemory = uint64(memInfo.free)
+			gpuInfo.ID = C.GoString(&memInfo.gpu_id[0])
+			gpuInfo.Compute = fmt.Sprintf("%d.%d", memInfo.major, memInfo.minor)
+			gpuInfo.MinimumMemory = cudaMinimumMemory
+			gpuInfo.DependencyPath = depPath
+			gpuInfo.Name = C.GoString(&memInfo.gpu_name[0])
+			gpuInfo.DriverMajor = driverMajor
+			gpuInfo.DriverMinor = driverMinor
 
-		// TODO potentially sort on our own algorithm instead of what the underlying GPU library does...
-		resp = append(resp, gpuInfo)
+			// TODO potentially sort on our own algorithm instead of what the underlying GPU library does...
+			resp = append(resp, gpuInfo)
+		}
 	}
 
 	// Then AMD
@@ -283,6 +296,7 @@ func FindGPULibs(baseLibName string, defaultPatterns []string) []string {
 		// Nvidia PhysX known to return bogus results
 		if strings.Contains(pattern, "PhysX") {
 			slog.Debug("skipping PhysX cuda library path", "path", pattern)
+			continue
 		}
 		// Ignore glob discovery errors
 		matches, _ := filepath.Glob(pattern)
@@ -348,6 +362,23 @@ func LoadNVCUDAMgmt(nvcudaLibPaths []string) (int, *C.nvcuda_handle_t, string) {
 	return 0, nil, ""
 }
 
+func LoadOneapiMgmt(oneapiLibPaths []string) (int, *C.oneapi_handle_t, string) {
+	var resp C.oneapi_init_resp_t
+	resp.oh.verbose = getVerboseState()
+	for _, libPath := range oneapiLibPaths {
+		lib := C.CString(libPath)
+		defer C.free(unsafe.Pointer(lib))
+		C.oneapi_init(lib, &resp)
+		if resp.err != nil {
+			slog.Debug("Unable to load oneAPI management library", "library", libPath, "error", C.GoString(resp.err))
+			C.free(unsafe.Pointer(resp.err))
+		} else {
+			return int(resp.num_devices), &resp.oh, libPath
+		}
+	}
+	return 0, nil, ""
+}
+
 func getVerboseState() C.uint16_t {
 	if envconfig.Debug {
 		return C.uint16_t(1)
@@ -368,6 +399,8 @@ func (l GpuInfoList) GetVisibleDevicesEnv() (string, string) {
 		return cudaGetVisibleDevicesEnv(l)
 	case "rocm":
 		return rocmGetVisibleDevicesEnv(l)
+	case "oneapi":
+		return oneapiGetVisibleDevicesEnv(l)
 	default:
 		slog.Debug("no filter required for library " + l[0].Library)
 		return "", ""

gpu/gpu_info.h

@@ -62,6 +62,7 @@ void cpu_check_ram(mem_info_t *resp);
 
 #include "gpu_info_cudart.h"
 #include "gpu_info_nvcuda.h"
+#include "gpu_info_oneapi.h"
 
 #endif  // __GPU_INFO_H__
 #endif  // __APPLE__

gpu/gpu_info_oneapi.c

@@ -0,0 +1,214 @@
+#ifndef __APPLE__
+
+#include "gpu_info_oneapi.h"
+
+#include <string.h>
+
+void oneapi_init(char *oneapi_lib_path, oneapi_init_resp_t *resp)
+{
+  ze_result_t ret;
+  resp->err = NULL;
+  const int buflen = 256;
+  char buf[buflen + 1];
+  int i;
+  struct lookup
+  {
+    char *s;
+    void **p;
+  } l[] = {
+      {"zesInit", (void *)&resp->oh.zesInit},
+      {"zesDriverGet", (void *)&resp->oh.zesDriverGet},
+      {"zesDeviceGet", (void *)&resp->oh.zesDeviceGet},
+      {"zesDeviceGetProperties", (void *)&resp->oh.zesDeviceGetProperties},
+      {"zesDeviceEnumMemoryModules",
+       (void *)&resp->oh.zesDeviceEnumMemoryModules},
+      {"zesMemoryGetProperties", (void *)&resp->oh.zesMemoryGetProperties},
+      {"zesMemoryGetState", (void *)&resp->oh.zesMemoryGetState},
+      {NULL, NULL},
+  };
+
+  resp->oh.handle = LOAD_LIBRARY(oneapi_lib_path, RTLD_LAZY);
+  if (!resp->oh.handle)
+  {
+    char *msg = LOAD_ERR();
+    snprintf(buf, buflen,
+             "Unable to load %s library to query for Intel GPUs: %s\n",
+             oneapi_lib_path, msg);
+    free(msg);
+    resp->err = strdup(buf);
+    return;
+  }
+
+  // TODO once we've squashed the remaining corner cases remove this log
+  LOG(resp->oh.verbose,
+      "wiring Level-Zero management library functions in %s\n",
+      oneapi_lib_path);
+
+  for (i = 0; l[i].s != NULL; i++)
+  {
+    // TODO once we've squashed the remaining corner cases remove this log
+    LOG(resp->oh.verbose, "dlsym: %s\n", l[i].s);
+
+    *l[i].p = LOAD_SYMBOL(resp->oh.handle, l[i].s);
+    if (!l[i].p)
+    {
+      resp->oh.handle = NULL;
+      char *msg = LOAD_ERR();
+      LOG(resp->oh.verbose, "dlerr: %s\n", msg);
+      UNLOAD_LIBRARY(resp->oh.handle);
+      snprintf(buf, buflen, "symbol lookup for %s failed: %s", l[i].s, msg);
+      free(msg);
+      resp->err = strdup(buf);
+      return;
+    }
+  }
+
+  ret = (*resp->oh.zesInit)(0);
+  if (ret != ZE_RESULT_SUCCESS)
+  {
+    LOG(resp->oh.verbose, "zesInit err: %d\n", ret);
+    UNLOAD_LIBRARY(resp->oh.handle);
+    resp->oh.handle = NULL;
+    snprintf(buf, buflen, "oneapi vram init failure: %d", ret);
+    resp->err = strdup(buf);
+  }
+
+  (*resp->oh.zesDriverGet)(&resp->num_devices, NULL);
+
+  return;
+}
+
+void oneapi_check_vram(oneapi_handle_t h, mem_info_t *resp)
+{
+  ze_result_t ret;
+  resp->err = NULL;
+  uint64_t totalMem = 0;
+  uint64_t usedMem = 0;
+  const int buflen = 256;
+  char buf[buflen + 1];
+  int i, d, m;
+
+  if (h.handle == NULL)
+  {
+    resp->err = strdup("Level-Zero handle not initialized");
+    return;
+  }
+
+  uint32_t driversCount = 0;
+  ret = (*h.zesDriverGet)(&driversCount, NULL);
+  if (ret != ZE_RESULT_SUCCESS)
+  {
+    snprintf(buf, buflen, "unable to get driver count: %d", ret);
+    resp->err = strdup(buf);
+    return;
+  }
+  LOG(h.verbose, "discovered %d Level-Zero drivers\n", driversCount);
+
+  zes_driver_handle_t *allDrivers =
+      malloc(driversCount * sizeof(zes_driver_handle_t));
+  (*h.zesDriverGet)(&driversCount, allDrivers);
+
+  resp->total = 0;
+  resp->free = 0;
+
+  for (d = 0; d < driversCount; d++)
+  {
+    uint32_t deviceCount = 0;
+    ret = (*h.zesDeviceGet)(allDrivers[d], &deviceCount, NULL);
+    if (ret != ZE_RESULT_SUCCESS)
+    {
+      snprintf(buf, buflen, "unable to get device count: %d", ret);
+      resp->err = strdup(buf);
+      free(allDrivers);
+      return;
+    }
+
+    LOG(h.verbose, "discovered %d Level-Zero devices\n", deviceCount);
+
+    zes_device_handle_t *devices =
+        malloc(deviceCount * sizeof(zes_device_handle_t));
+    (*h.zesDeviceGet)(allDrivers[d], &deviceCount, devices);
+
+    for (i = 0; i < deviceCount; i++)
+    {
+      zes_device_ext_properties_t ext_props;
+      ext_props.stype = ZES_STRUCTURE_TYPE_DEVICE_EXT_PROPERTIES;
+      ext_props.pNext = NULL;
+
+      zes_device_properties_t props;
+      props.stype = ZES_STRUCTURE_TYPE_DEVICE_PROPERTIES;
+      props.pNext = &ext_props;
+
+      ret = (*h.zesDeviceGetProperties)(devices[i], &props);
+      if (ret != ZE_RESULT_SUCCESS)
+      {
+        snprintf(buf, buflen, "unable to get device properties: %d", ret);
+        resp->err = strdup(buf);
+        free(allDrivers);
+        free(devices);
+        return;
+      }
+
+      if (h.verbose)
+      {
+        // When in verbose mode, report more information about
+        // the card we discover.
+        LOG(h.verbose, "[%d] oneAPI device name: %s\n", i,
+            props.modelName);
+        LOG(h.verbose, "[%d] oneAPI brand: %s\n", i,
+            props.brandName);
+        LOG(h.verbose, "[%d] oneAPI vendor: %s\n", i,
+            props.vendorName);
+        LOG(h.verbose, "[%d] oneAPI S/N: %s\n", i,
+            props.serialNumber);
+        LOG(h.verbose, "[%d] oneAPI board number: %s\n", i,
+            props.boardNumber);
+      }
+
+      uint32_t memCount = 0;
+      ret = (*h.zesDeviceEnumMemoryModules)(devices[i], &memCount, NULL);
+      if (ret != ZE_RESULT_SUCCESS)
+      {
+        snprintf(buf, buflen,
+                 "unable to enumerate Level-Zero memory modules: %d", ret);
+        resp->err = strdup(buf);
+        free(allDrivers);
+        free(devices);
+        return;
+      }
+
+      LOG(h.verbose, "discovered %d Level-Zero memory modules\n", memCount);
+
+      zes_mem_handle_t *mems = malloc(memCount * sizeof(zes_mem_handle_t));
+      (*h.zesDeviceEnumMemoryModules)(devices[i], &memCount, mems);
+
+      for (m = 0; m < memCount; m++)
+      {
+        zes_mem_state_t state;
+        state.stype = ZES_STRUCTURE_TYPE_MEM_STATE;
+        state.pNext = NULL;
+        ret = (*h.zesMemoryGetState)(mems[m], &state);
+        if (ret != ZE_RESULT_SUCCESS)
+        {
+          snprintf(buf, buflen, "unable to get memory state: %d", ret);
+          resp->err = strdup(buf);
+          free(allDrivers);
+          free(devices);
+          free(mems);
+          return;
+        }
+
+        resp->total += state.size;
+        resp->free += state.free;
+      }
+
+      free(mems);
+    }
+
+    free(devices);
+  }
+
+  free(allDrivers);
+}
+
+#endif // __APPLE__

gpu/gpu_info_oneapi.h

@@ -0,0 +1,211 @@
+#ifndef __APPLE__
+#ifndef __GPU_INFO_ONEAPI_H__
+#define __GPU_INFO_ONEAPI_H__
+#include "gpu_info.h"
+
+#define ZE_MAX_DEVICE_NAME 256
+#define ZE_MAX_DEVICE_UUID_SIZE 16
+#define ZES_STRING_PROPERTY_SIZE 64
+#define ZE_BIT(_i) (1 << _i)
+
+// Just enough typedef's to dlopen/dlsym for memory information
+typedef enum ze_result_t
+{
+  ZE_RESULT_SUCCESS = 0,
+  // Other values omitted for now...
+} ze_result_t;
+
+typedef uint8_t ze_bool_t;
+typedef struct _zes_driver_handle_t *zes_driver_handle_t;
+typedef struct _zes_device_handle_t *zes_device_handle_t;
+typedef struct _zes_mem_handle_t *zes_mem_handle_t;
+
+typedef enum _ze_structure_type_t
+{
+  ZE_STRUCTURE_TYPE_FORCE_UINT32 = 0x7fffffff
+} ze_structure_type_t;
+
+typedef enum _zes_structure_type_t
+{
+  ZES_STRUCTURE_TYPE_DEVICE_PROPERTIES = 0x1,
+  ZES_STRUCTURE_TYPE_MEM_PROPERTIES = 0xb,
+  ZES_STRUCTURE_TYPE_MEM_STATE = 0x1e,
+  ZES_STRUCTURE_TYPE_DEVICE_EXT_PROPERTIES = 0x2d,
+  ZES_STRUCTURE_TYPE_FORCE_UINT32 = 0x7fffffff
+} zes_structure_type_t;
+
+typedef enum _zes_mem_type_t
+{
+  ZES_MEM_TYPE_FORCE_UINT32 = 0x7fffffff
+} zes_mem_type_t;
+
+typedef enum _zes_mem_loc_t
+{
+  ZES_MEM_LOC_SYSTEM = 0,
+  ZES_MEM_LOC_DEVICE = 1,
+  ZES_MEM_LOC_FORCE_UINT32 = 0x7fffffff
+} zes_mem_loc_t;
+
+typedef enum _zes_mem_health_t
+{
+  ZES_MEM_HEALTH_FORCE_UINT32 = 0x7fffffff
+} zes_mem_health_t;
+
+typedef struct _ze_device_uuid_t
+{
+  uint8_t id[ZE_MAX_DEVICE_UUID_SIZE];
+} ze_device_uuid_t;
+
+typedef struct _zes_uuid_t
+{
+  uint8_t id[ZE_MAX_DEVICE_UUID_SIZE];
+} zes_uuid_t;
+
+typedef enum _ze_device_type_t
+{
+  ZE_DEVICE_TYPE_GPU = 1,
+  ZE_DEVICE_TYPE_CPU = 2,
+  ZE_DEVICE_TYPE_FPGA = 3,
+  ZE_DEVICE_TYPE_MCA = 4,
+  ZE_DEVICE_TYPE_VPU = 5,
+  ZE_DEVICE_TYPE_FORCE_UINT32 = 0x7fffffff
+} ze_device_type_t;
+
+typedef enum _zes_device_type_t
+{
+  ZES_DEVICE_TYPE_GPU = 1,
+  ZES_DEVICE_TYPE_CPU = 2,
+  ZES_DEVICE_TYPE_FPGA = 3,
+  ZES_DEVICE_TYPE_MCA = 4,
+  ZES_DEVICE_TYPE_VPU = 5,
+  ZES_DEVICE_TYPE_FORCE_UINT32 = 0x7fffffff
+} zes_device_type_t;
+
+typedef uint32_t ze_device_property_flags_t;
+typedef enum _ze_device_property_flag_t
+{
+  ZE_DEVICE_PROPERTY_FLAG_INTEGRATED = ZE_BIT(0),
+  ZE_DEVICE_PROPERTY_FLAG_SUBDEVICE = ZE_BIT(1),
+  ZE_DEVICE_PROPERTY_FLAG_ECC = ZE_BIT(2),
+  ZE_DEVICE_PROPERTY_FLAG_ONDEMANDPAGING = ZE_BIT(3),
+  ZE_DEVICE_PROPERTY_FLAG_FORCE_UINT32 = 0x7fffffff
+} ze_device_property_flag_t;
+
+typedef uint32_t zes_device_property_flags_t;
+typedef enum _zes_device_property_flag_t
+{
+  ZES_DEVICE_PROPERTY_FLAG_INTEGRATED = ZE_BIT(0),
+  ZES_DEVICE_PROPERTY_FLAG_SUBDEVICE = ZE_BIT(1),
+  ZES_DEVICE_PROPERTY_FLAG_ECC = ZE_BIT(2),
+  ZES_DEVICE_PROPERTY_FLAG_ONDEMANDPAGING = ZE_BIT(3),
+  ZES_DEVICE_PROPERTY_FLAG_FORCE_UINT32 = 0x7fffffff
+} zes_device_property_flag_t;
+
+typedef struct _ze_device_properties_t
+{
+  ze_structure_type_t stype;
+  void *pNext;
+  ze_device_type_t type;
+  uint32_t vendorId;
+  uint32_t deviceId;
+  ze_device_property_flags_t flags;
+  uint32_t subdeviceId;
+  uint32_t coreClockRate;
+  uint64_t maxMemAllocSize;
+  uint32_t maxHardwareContexts;
+  uint32_t maxCommandQueuePriority;
+  uint32_t numThreadsPerEU;
+  uint32_t physicalEUSimdWidth;
+  uint32_t numEUsPerSubslice;
+  uint32_t numSubslicesPerSlice;
+  uint32_t numSlices;
+  uint64_t timerResolution;
+  uint32_t timestampValidBits;
+  uint32_t kernelTimestampValidBits;
+  ze_device_uuid_t uuid;
+  char name[ZE_MAX_DEVICE_NAME];
+} ze_device_properties_t;
+
+typedef struct _zes_device_properties_t
+{
+  zes_structure_type_t stype;
+  void *pNext;
+  ze_device_properties_t core;
+  uint32_t numSubdevices;
+  char serialNumber[ZES_STRING_PROPERTY_SIZE];
+  char boardNumber[ZES_STRING_PROPERTY_SIZE];
+  char brandName[ZES_STRING_PROPERTY_SIZE];
+  char modelName[ZES_STRING_PROPERTY_SIZE];
+  char vendorName[ZES_STRING_PROPERTY_SIZE];
+  char driverVersion[ZES_STRING_PROPERTY_SIZE];
+} zes_device_properties_t;
+
+typedef struct _zes_device_ext_properties_t
+{
+  zes_structure_type_t stype;
+  void *pNext;
+  zes_uuid_t uuid;
+  zes_device_type_t type;
+  zes_device_property_flags_t flags;
+} zes_device_ext_properties_t;
+
+typedef struct _zes_mem_properties_t
+{
+  zes_structure_type_t stype;
+  void *pNext;
+  zes_mem_type_t type;
+  ze_bool_t onSubdevice;
+  uint32_t subdeviceId;
+  zes_mem_loc_t location;
+  uint64_t physicalSize;
+  int32_t busWidth;
+  int32_t numChannels;
+} zes_mem_properties_t;
+
+typedef struct _zes_mem_state_t
+{
+  zes_structure_type_t stype;
+  const void *pNext;
+  zes_mem_health_t health;
+  uint64_t free;
+  uint64_t size;
+} zes_mem_state_t;
+
+typedef struct oneapi_handle
+{
+  void *handle;
+  uint16_t verbose;
+  ze_result_t (*zesInit)(int);
+  ze_result_t (*zesDriverGet)(uint32_t *pCount, zes_driver_handle_t *phDrivers);
+  ze_result_t (*zesDeviceGet)(zes_driver_handle_t hDriver, uint32_t *pCount,
+                              zes_device_handle_t *phDevices);
+  ze_result_t (*zesDeviceGetProperties)(zes_device_handle_t hDevice,
+                                        zes_device_properties_t *pProperties);
+  ze_result_t (*zesDeviceEnumMemoryModules)(zes_device_handle_t hDevice,
+                                            uint32_t *pCount,
+                                            zes_mem_handle_t *phMemory);
+  ze_result_t (*zesMemoryGetProperties)(zes_mem_handle_t hMemory,
+                                        zes_mem_properties_t *pProperties);
+  ze_result_t (*zesMemoryGetState)(zes_mem_handle_t hMemory,
+                                   zes_mem_state_t *pState);
+
+} oneapi_handle_t;
+
+typedef struct oneapi_init_resp
+{
+  char *err; // If err is non-null handle is invalid
+  int num_devices;
+  oneapi_handle_t oh;
+} oneapi_init_resp_t;
+
+typedef struct oneapi_version_resp
+{
+  ze_result_t status;
+  char *str; // Contains version or error string if status != 0
+} oneapi_version_resp_t;
+
+void oneapi_init(char *oneapi_lib_path, oneapi_init_resp_t *resp);
+void oneapi_check_vram(oneapi_handle_t rh, mem_info_t *resp);
+
+#endif // __GPU_INFO_INTEL_H__
+#endif // __APPLE__

gpu/gpu_oneapi.go

@@ -0,0 +1,21 @@
+//go:build linux || windows
+
+package gpu
+
+import (
+	"log/slog"
+	"strings"
+)
+
+func oneapiGetVisibleDevicesEnv(gpuInfo []GpuInfo) (string, string) {
+	ids := []string{}
+	for _, info := range gpuInfo {
+		if info.Library != "oneapi" {
+			// TODO shouldn't happen if things are wired correctly...
+			slog.Debug("oneapiGetVisibleDevicesEnv skipping over non-sycl device", "library", info.Library)
+			continue
+		}
+		ids = append(ids, info.ID)
+	}
+	return "ONEAPI_DEVICE_SELECTOR", "level_zero:" + strings.Join(ids, ",")
+}

gpu/gpu_test.go

@@ -5,11 +5,12 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestBasicGetGPUInfo(t *testing.T) {
 	info := GetGPUInfo()
-	assert.Greater(t, len(info), 0)
+	assert.NotEmpty(t, len(info))
 	assert.Contains(t, "cuda rocm cpu metal", info[0].Library)
 	if info[0].Library != "cpu" {
 		assert.Greater(t, info[0].TotalMemory, uint64(0))
@@ -19,7 +20,7 @@ func TestBasicGetGPUInfo(t *testing.T) {
 
 func TestCPUMemInfo(t *testing.T) {
 	info, err := GetCPUMem()
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	switch runtime.GOOS {
 	case "darwin":
 		t.Skip("CPU memory not populated on darwin")

integration/max_queue_test.go

@@ -19,6 +19,11 @@ import (
 )
 
 func TestMaxQueue(t *testing.T) {
+	if os.Getenv("OLLAMA_TEST_EXISTING") != "" {
+		t.Skip("Max Queue test requires spawing a local server so we can adjust the queue size")
+		return
+	}
+
 	// Note: This test can be quite slow when running in CPU mode, so keep the threadCount low unless your on GPU
 	// Also note that by default Darwin can't sustain > ~128 connections without adjusting limits
 	threadCount := 32
@@ -109,9 +114,9 @@ func TestMaxQueue(t *testing.T) {
 	slog.Info("generate done, waiting for embeds")
 	embedwg.Wait()
 
+	slog.Info("embeds completed", "success", succesCount, "busy", busyCount, "reset", resetByPeerCount, "canceled", canceledCount)
 	require.Equal(t, resetByPeerCount, 0, "Connections reset by peer, have you updated your fd and socket limits?")
 	require.True(t, busyCount > 0, "no requests hit busy error but some should have")
 	require.True(t, canceledCount == 0, "no requests should have been canceled due to timeout")
 
-	slog.Info("embeds completed", "success", succesCount, "busy", busyCount, "reset", resetByPeerCount, "canceled", canceledCount)
 }

llm/ext_server/server.cpp

@@ -140,7 +140,6 @@ struct server_slot {
     std::vector<llama_token> cache_tokens;
     std::vector<completion_token_output> generated_token_probs;
 
-    bool infill = false;
     bool embedding = false;
     bool has_next_token = true;
     bool truncated = false;
@@ -187,7 +186,6 @@ struct server_slot {
         n_past                 = 0;
         n_sent_text            = 0;
         n_sent_token_probs     = 0;
-        infill                 = false;
         ga_i                   = 0;
         n_past_se              = 0;
 
@@ -334,6 +332,7 @@ struct server_metrics {
 struct llama_server_context
 {
     llama_model *model = nullptr;
+    float modelProgress = 0.0;
     llama_context *ctx = nullptr;
 
     clip_ctx *clp_ctx = nullptr;
@@ -360,7 +359,6 @@ struct llama_server_context
 
     // slots / clients
     std::vector<server_slot> slots;
-    json default_generation_settings_for_props;
 
     llama_server_queue    queue_tasks;
     llama_server_response queue_results;
@@ -484,9 +482,6 @@ struct llama_server_context
             slots.push_back(slot);
         }
 
-        default_generation_settings_for_props = get_formated_generation(slots.front());
-        default_generation_settings_for_props["seed"] = -1;
-
         batch = llama_batch_init(n_ctx, 0, params.n_parallel);
     }
 
@@ -585,7 +580,7 @@ struct llama_server_context
         slot->sparams.mirostat_eta      = json_value(data, "mirostat_eta",      default_sparams.mirostat_eta);
         slot->sparams.penalize_nl       = json_value(data, "penalize_nl",       default_sparams.penalize_nl);
         slot->params.n_keep             = json_value(data, "n_keep",            slot->params.n_keep);
-        slot->params.seed               = json_value(data, "seed",              default_params.seed);
+        slot->sparams.seed              = json_value(data, "seed",              default_params.seed);
         slot->sparams.grammar           = json_value(data, "grammar",           default_sparams.grammar);
         slot->sparams.n_probs           = json_value(data, "n_probs",           default_sparams.n_probs);
         slot->sparams.min_keep          = json_value(data, "min_keep",          default_sparams.min_keep);
@@ -599,16 +594,6 @@ struct llama_server_context
             slot->params.n_predict = slot->n_predict;
         }
 
-        // infill
-        if (data.count("input_prefix") != 0)
-        {
-            slot->params.input_prefix = data["input_prefix"];
-        }
-        else
-        {
-            slot->params.input_prefix = "";
-        }
-
         if (data.count("input_suffix") != 0)
         {
             slot->params.input_suffix = data["input_suffix"];
@@ -737,7 +722,7 @@ struct llama_server_context
                     sampler_names.emplace_back(sampler_name);
                 }
             }
-            slot->sparams.samplers_sequence = sampler_types_from_names(sampler_names, false);
+            slot->sparams.samplers_sequence = llama_sampling_types_from_names(sampler_names, false);
         }
         else
         {
@@ -822,7 +807,6 @@ struct llama_server_context
             llama_sampling_free(slot->ctx_sampling);
         }
         slot->ctx_sampling = llama_sampling_init(slot->sparams);
-        llama_set_rng_seed(ctx, slot->params.seed);
         slot->command = LOAD_PROMPT;
 
         all_slots_are_idle = false;
@@ -846,7 +830,7 @@ struct llama_server_context
         system_tokens.clear();
 
         if (!system_prompt.empty()) {
-            system_tokens = ::llama_tokenize(ctx, system_prompt, add_bos_token);
+            system_tokens = ::llama_tokenize(ctx, system_prompt, true);
 
             llama_batch_clear(batch);
 
@@ -896,15 +880,6 @@ struct llama_server_context
         system_need_update = true;
     }
 
-    void system_prompt_process(const json &sys_props) {
-        system_prompt  = sys_props.value("prompt", "");
-        name_user      = sys_props.value("anti_prompt", "");
-        name_assistant = sys_props.value("assistant_name", "");
-
-
-        system_prompt_notify();
-    }
-
     static size_t find_stopping_strings(const std::string &text, const size_t last_token_size,
                                         const stop_type type, server_slot &slot)
     {
@@ -1095,7 +1070,7 @@ struct llama_server_context
         std::vector<std::string> samplers_sequence;
         for (const auto &sampler_type : slot.sparams.samplers_sequence)
         {
-            samplers_sequence.emplace_back(sampler_type_to_name_string(sampler_type));
+            samplers_sequence.emplace_back(llama_sampling_type_to_str(sampler_type));
         }
 
         return json {
@@ -1262,13 +1237,12 @@ struct llama_server_context
         queue_results.send(res);
     }
 
-    void request_completion(int task_id, json data, bool infill, bool embedding, int multitask_id)
+    void request_completion(int task_id, json data, bool embedding, int multitask_id)
     {
         task_server task;
         task.id = task_id;
         task.target_id = 0;
         task.data = std::move(data);
-        task.infill_mode = infill;
         task.embedding_mode = embedding;
         task.type = TASK_TYPE_COMPLETION;
         task.multitask_id = multitask_id;
@@ -1414,8 +1388,8 @@ struct llama_server_context
             json subtask_data = multiprompt_task.data;
             subtask_data["prompt"] = subtask_data["prompt"][i];
 
-            // subtasks inherit everything else (infill mode, embedding mode, etc.)
-            request_completion(subtask_ids[i], subtask_data, multiprompt_task.infill_mode, multiprompt_task.embedding_mode, multitask_id);
+            // subtasks inherit everything else (embedding mode, etc.)
+            request_completion(subtask_ids[i], subtask_data, multiprompt_task.embedding_mode, multitask_id);
         }
     }
 
@@ -1433,26 +1407,8 @@ struct llama_server_context
                     break;
                 }
 
-                if (task.data.contains("system_prompt"))
-                {
-                    if (!all_slots_are_idle) {
-                        send_error(task, "system prompt can only be updated when all slots are idle");
-                        break;
-                    }
-                    system_prompt_process(task.data["system_prompt"]);
-
-                    // reset cache_tokens for all slots
-                    for (server_slot &slot : slots)
-                    {
-                        slot.cache_tokens.clear();
-                        slot.n_past    = 0;
-                        slot.n_past_se = 0;
-                    }
-                }
-
                 slot->reset();
 
-                slot->infill       = task.infill_mode;
                 slot->embedding    = task.embedding_mode;
                 slot->task_id      = task.id;
                 slot->multitask_id = task.multitask_id;
@@ -1678,8 +1634,7 @@ struct llama_server_context
                 const bool has_prompt = slot.prompt.is_array() || (slot.prompt.is_string() && !slot.prompt.get<std::string>().empty()) || !slot.images.empty();
 
                 // empty prompt passed -> release the slot and send empty response
-                // note: infill mode allows empty prompt
-                if (slot.state == IDLE && slot.command == LOAD_PROMPT && !has_prompt && !slot.infill)
+                if (slot.state == IDLE && slot.command == LOAD_PROMPT && !has_prompt)
                 {
                     slot.release();
                     slot.print_timings();
@@ -1696,33 +1651,7 @@ struct llama_server_context
                     slot.t_start_process_prompt = ggml_time_us();
                     slot.t_start_genereration = 0;
 
-                    if (slot.infill)
-                    {
-                        bool suff_rm_leading_spc = true;
-                        if (params.input_suffix.find_first_of(' ') == 0 && params.input_suffix.size() > 1)
-                        {
-                            params.input_suffix.erase(0, 1);
-                            suff_rm_leading_spc = false;
-                        }
-                        auto prefix_tokens = tokenize(slot.params.input_prefix, false);
-                        auto suffix_tokens = tokenize(slot.params.input_suffix, false);
-
-                        const int space_token = 29871; // TODO: this should not be hardcoded
-                        if (suff_rm_leading_spc && !suffix_tokens.empty() && suffix_tokens[0] == space_token) {
-                            suffix_tokens.erase(suffix_tokens.begin());
-                        }
-
-                        prefix_tokens.insert(prefix_tokens.begin(), llama_token_prefix(model));
-                        prefix_tokens.insert(prefix_tokens.begin(), llama_token_bos(model)); // always add BOS
-                        prefix_tokens.insert(prefix_tokens.end(),   llama_token_suffix(model));
-                        prefix_tokens.insert(prefix_tokens.end(),   suffix_tokens.begin(), suffix_tokens.end());
-                        prefix_tokens.push_back(llama_token_middle(model));
-                        prompt_tokens = prefix_tokens;
-                    }
-                    else
-                    {
-                        prompt_tokens = tokenize(slot.prompt, system_prompt.empty() && add_bos_token);  // add BOS if there isn't system prompt
-                    }
+                    prompt_tokens = tokenize(slot.prompt, system_prompt.empty());  // add BOS if there isn't system prompt
 
                     slot.n_prompt_tokens = prompt_tokens.size();
 
@@ -2104,6 +2033,7 @@ static void server_print_usage(const char *argv0, const gpt_params &params,
     printf("  --embedding               enable embedding vector output (default: %s)\n", params.embedding ? "enabled" : "disabled");
     printf("  -np N, --parallel N       number of slots for process requests (default: %d)\n", params.n_parallel);
     printf("  -cb, --cont-batching      enable continuous batching (a.k.a dynamic batching) (default: disabled)\n");
+    printf("  -fa, --flash-attn         enable Flash Attention (default: %s)\n", params.flash_attn ? "enabled" : "disabled");
     printf("  -spf FNAME, --system-prompt-file FNAME\n");
     printf("                            set a file to load a system prompt (initial prompt of all slots), this is useful for chat applications.\n");
     printf("  -ctk TYPE, --cache-type-k TYPE\n");
@@ -2128,8 +2058,7 @@ static void server_print_usage(const char *argv0, const gpt_params &params,
     printf("\n");
 }
 
-static void server_params_parse(int argc, char **argv, server_params &sparams,
-                                gpt_params &params, llama_server_context& llama)
+static void server_params_parse(int argc, char **argv, server_params &sparams, gpt_params &params)
 {
     gpt_params default_params;
     server_params default_sparams;
@@ -2501,7 +2430,8 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,
         {
             params.use_mmap = false;
         }
-        else if (arg == "--numa") {
+        else if (arg == "--numa")
+        {
             if (++i >= argc) {
                 invalid_param = true;
                 break;
@@ -2521,6 +2451,10 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,
         {
             params.cont_batching = true;
         }
+        else if (arg == "-fa" || arg == "--flash-attn")
+        {
+            params.flash_attn = true;
+        }
         else if (arg == "-np" || arg == "--parallel")
         {
             if (++i >= argc)
@@ -2529,7 +2463,8 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,
                 break;
             }
             params.n_parallel = std::stoi(argv[i]);
-        } else if (arg == "-n" || arg == "--n-predict")
+        }
+        else if (arg == "-n" || arg == "--n-predict")
         {
             if (++i >= argc)
             {
@@ -2537,26 +2472,6 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,
                 break;
             }
             params.n_predict = std::stoi(argv[i]);
-        } else if (arg == "-spf" || arg == "--system-prompt-file")
-        {
-            if (++i >= argc)
-            {
-                invalid_param = true;
-                break;
-            }
-            std::ifstream file(argv[i]);
-            if (!file) {
-                fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);
-                invalid_param = true;
-                break;
-            }
-            std::string systm_content;
-            std::copy(
-                std::istreambuf_iterator<char>(file),
-                std::istreambuf_iterator<char>(),
-                std::back_inserter(systm_content)
-            );
-            llama.system_prompt_process(json::parse(systm_content));
         }
         else if (arg == "-ctk" || arg == "--cache-type-k") {
             params.cache_type_k = argv[++i];
@@ -2771,6 +2686,12 @@ inline void signal_handler(int signal) {
     shutdown_handler(signal);
 }
 
+static bool update_load_progress(float progress, void *data)
+{
+    ((llama_server_context*)data)->modelProgress = progress;
+    return true;
+}
+
 #if defined(_WIN32)
 char* wchar_to_char(const wchar_t* wstr) {
     if (wstr == nullptr) return nullptr;
@@ -2803,7 +2724,7 @@ int main(int argc, char **argv) {
     // struct that contains llama context and inference
     llama_server_context llama;
 
-    server_params_parse(argc, argv, sparams, params, llama);
+    server_params_parse(argc, argv, sparams, params);
 
     if (params.model_alias == "unknown")
     {
@@ -2876,7 +2797,9 @@ int main(int argc, char **argv) {
                 break;
             }
             case SERVER_STATE_LOADING_MODEL:
-                res.set_content(R"({"status": "loading model"})", "application/json");
+                char buf[128];
+                snprintf(&buf[0], 128, R"({"status": "loading model", "progress": %0.2f})", llama.modelProgress);
+                res.set_content(buf, "application/json");
                 res.status = 503; // HTTP Service Unavailable
                 break;
             case SERVER_STATE_ERROR:
@@ -3071,6 +2994,9 @@ int main(int argc, char **argv) {
             });
 
     // load the model
+    params.progress_callback = update_load_progress;
+    params.progress_callback_user_data = (void*)&llama;
+
     if (!llama.load_model(params))
     {
         state.store(SERVER_STATE_ERROR);
@@ -3130,7 +3056,7 @@ int main(int argc, char **argv) {
                 json data = json::parse(req.body);
                 const int task_id = llama.queue_tasks.get_new_id();
                 llama.queue_results.add_waiting_task_id(task_id);
-                llama.request_completion(task_id, data, false, false, -1);
+                llama.request_completion(task_id, data, false, -1);
                 if (!json_value(data, "stream", false)) {
                     std::string completion_text;
                     task_result result = llama.queue_results.recv(task_id);
@@ -3252,7 +3178,7 @@ int main(int argc, char **argv) {
                 // create and queue the task
                 const int task_id = llama.queue_tasks.get_new_id();
                 llama.queue_results.add_waiting_task_id(task_id);
-                llama.request_completion(task_id, { {"prompt", prompt}, { "n_predict", 0}, {"image_data", image_data} }, false, true, -1);
+                llama.request_completion(task_id, { {"prompt", prompt}, { "n_predict", 0}, {"image_data", image_data} }, true, -1);
 
                 // get the result
                 task_result result = llama.queue_results.recv(task_id);

llm/filetype.go

@@ -27,8 +27,16 @@ const (
 	fileTypeIQ2_XXS
 	fileTypeIQ2_XS
 	fileTypeQ2_K_S
-	fileTypeQ3_K_XS
+	fileTypeIQ3_XS
 	fileTypeIQ3_XXS
+	fileTypeIQ1_S
+	fileTypeIQ4_NL
+	fileTypeIQ3_S
+	fileTypeIQ2_S
+	fileTypeIQ4_XS
+	fileTypeIQ2_M
+	fileTypeIQ1_M
+	fileTypeBF16
 
 	fileTypeUnknown
 )
@@ -75,10 +83,26 @@ func ParseFileType(s string) (fileType, error) {
 		return fileTypeIQ2_XS, nil
 	case "Q2_K_S":
 		return fileTypeQ2_K_S, nil
-	case "Q3_K_XS":
-		return fileTypeQ3_K_XS, nil
+	case "IQ3_XS":
+		return fileTypeIQ3_XS, nil
 	case "IQ3_XXS":
 		return fileTypeIQ3_XXS, nil
+	case "IQ1_S":
+		return fileTypeIQ1_S, nil
+	case "IQ4_NL":
+		return fileTypeIQ4_NL, nil
+	case "IQ3_S":
+		return fileTypeIQ3_S, nil
+	case "IQ2_S":
+		return fileTypeIQ2_S, nil
+	case "IQ4_XS":
+		return fileTypeIQ4_XS, nil
+	case "IQ2_M":
+		return fileTypeIQ2_M, nil
+	case "IQ1_M":
+		return fileTypeIQ1_M, nil
+	case "BF16":
+		return fileTypeBF16, nil
 	default:
 		return fileTypeUnknown, fmt.Errorf("unknown fileType: %s", s)
 	}
@@ -126,10 +150,26 @@ func (t fileType) String() string {
 		return "IQ2_XS"
 	case fileTypeQ2_K_S:
 		return "Q2_K_S"
-	case fileTypeQ3_K_XS:
-		return "Q3_K_XS"
+	case fileTypeIQ3_XS:
+		return "IQ3_XS"
 	case fileTypeIQ3_XXS:
 		return "IQ3_XXS"
+	case fileTypeIQ1_S:
+		return "IQ1_S"
+	case fileTypeIQ4_NL:
+		return "IQ4_NL"
+	case fileTypeIQ3_S:
+		return "IQ3_S"
+	case fileTypeIQ2_S:
+		return "IQ2_S"
+	case fileTypeIQ4_XS:
+		return "IQ4_XS"
+	case fileTypeIQ2_M:
+		return "IQ2_M"
+	case fileTypeIQ1_M:
+		return "IQ1_M"
+	case fileTypeBF16:
+		return "BF16"
 	default:
 		return "unknown"
 	}

llm/generate/gen_darwin.sh

@@ -32,42 +32,43 @@ case "${GOARCH}" in
     echo "Building static library"
     build
 
+    if [ -z "$OLLAMA_SKIP_CPU_GENERATE" ]; then
+        #
+        # CPU first for the default library, set up as lowest common denominator for maximum compatibility (including Rosetta)
+        #
+        init_vars
+        CMAKE_DEFS="${COMMON_CPU_DEFS} -DLLAMA_ACCELERATE=off -DLLAMA_AVX=off -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off ${CMAKE_DEFS}"
+        BUILD_DIR="../build/darwin/${ARCH}/cpu"
+        echo "Building LCD CPU"
+        build
+        sign ${BUILD_DIR}/bin/ollama_llama_server
+        compress
 
-    #
-    # CPU first for the default library, set up as lowest common denominator for maximum compatibility (including Rosetta)
-    #
-    init_vars
-    CMAKE_DEFS="${COMMON_CPU_DEFS} -DLLAMA_ACCELERATE=off -DLLAMA_AVX=off -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off ${CMAKE_DEFS}"
-    BUILD_DIR="../build/darwin/${ARCH}/cpu"
-    echo "Building LCD CPU"
-    build
-    sign ${BUILD_DIR}/bin/ollama_llama_server
-    compress
+        #
+        # ~2011 CPU Dynamic library with more capabilities turned on to optimize performance
+        # Approximately 400% faster than LCD on same CPU
+        #
+        init_vars
+        CMAKE_DEFS="${COMMON_CPU_DEFS} -DLLAMA_ACCELERATE=off -DLLAMA_AVX=on -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off ${CMAKE_DEFS}"
+        BUILD_DIR="../build/darwin/${ARCH}/cpu_avx"
+        echo "Building AVX CPU"
+        build
+        sign ${BUILD_DIR}/bin/ollama_llama_server
+        compress
 
-    #
-    # ~2011 CPU Dynamic library with more capabilities turned on to optimize performance
-    # Approximately 400% faster than LCD on same CPU
-    #
-    init_vars
-    CMAKE_DEFS="${COMMON_CPU_DEFS} -DLLAMA_ACCELERATE=off -DLLAMA_AVX=on -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off ${CMAKE_DEFS}"
-    BUILD_DIR="../build/darwin/${ARCH}/cpu_avx"
-    echo "Building AVX CPU"
-    build
-    sign ${BUILD_DIR}/bin/ollama_llama_server
-    compress
-
-    #
-    # ~2013 CPU Dynamic library
-    # Approximately 10% faster than AVX on same CPU
-    #
-    init_vars
-    CMAKE_DEFS="${COMMON_CPU_DEFS} -DLLAMA_ACCELERATE=on -DLLAMA_AVX=on -DLLAMA_AVX2=on -DLLAMA_AVX512=off -DLLAMA_FMA=on -DLLAMA_F16C=on ${CMAKE_DEFS}"
-    BUILD_DIR="../build/darwin/${ARCH}/cpu_avx2"
-    echo "Building AVX2 CPU"
-    EXTRA_LIBS="${EXTRA_LIBS} -framework Accelerate -framework Foundation"
-    build
-    sign ${BUILD_DIR}/bin/ollama_llama_server
-    compress
+        #
+        # ~2013 CPU Dynamic library
+        # Approximately 10% faster than AVX on same CPU
+        #
+        init_vars
+        CMAKE_DEFS="${COMMON_CPU_DEFS} -DLLAMA_ACCELERATE=on -DLLAMA_AVX=on -DLLAMA_AVX2=on -DLLAMA_AVX512=off -DLLAMA_FMA=on -DLLAMA_F16C=on ${CMAKE_DEFS}"
+        BUILD_DIR="../build/darwin/${ARCH}/cpu_avx2"
+        echo "Building AVX2 CPU"
+        EXTRA_LIBS="${EXTRA_LIBS} -framework Accelerate -framework Foundation"
+        build
+        sign ${BUILD_DIR}/bin/ollama_llama_server
+        compress
+    fi
     ;;
 "arm64")
 
@@ -79,13 +80,15 @@ case "${GOARCH}" in
     echo "Building static library"
     build
 
-    init_vars
-    CMAKE_DEFS="${COMMON_DARWIN_DEFS} -DLLAMA_ACCELERATE=on -DCMAKE_SYSTEM_PROCESSOR=${ARCH} -DCMAKE_OSX_ARCHITECTURES=${ARCH} -DLLAMA_METAL=on ${CMAKE_DEFS}"
-    BUILD_DIR="../build/darwin/${ARCH}/metal"
-    EXTRA_LIBS="${EXTRA_LIBS} -framework Accelerate -framework Foundation -framework Metal -framework MetalKit -framework MetalPerformanceShaders"
-    build
-    sign ${BUILD_DIR}/bin/ollama_llama_server
-    compress
+    if [ -z "$OLLAMA_SKIP_METAL_GENERATE" ]; then
+        init_vars
+        CMAKE_DEFS="${COMMON_DARWIN_DEFS} -DLLAMA_ACCELERATE=on -DCMAKE_SYSTEM_PROCESSOR=${ARCH} -DCMAKE_OSX_ARCHITECTURES=${ARCH} -DLLAMA_METAL=on ${CMAKE_DEFS}"
+        BUILD_DIR="../build/darwin/${ARCH}/metal"
+        EXTRA_LIBS="${EXTRA_LIBS} -framework Accelerate -framework Foundation -framework Metal -framework MetalKit -framework MetalPerformanceShaders"
+        build
+        sign ${BUILD_DIR}/bin/ollama_llama_server
+        compress
+    fi
     ;;
 *)
     echo "GOARCH must be set"

llm/generate/gen_linux.sh

@@ -156,7 +156,7 @@ if [ -z "${CUDART_LIB_DIR}" ]; then
     CUDART_LIB_DIR="${CUDA_LIB_DIR}"
 fi
 
-if [ -d "${CUDA_LIB_DIR}" ]; then
+if [ -z "${OLLAMA_SKIP_CUDA_GENERATE}" -a -d "${CUDA_LIB_DIR}" ]; then
     echo "CUDA libraries detected - building dynamic CUDA library"
     init_vars
     CUDA_MAJOR=$(ls "${CUDA_LIB_DIR}"/libcudart.so.* | head -1 | cut -f3 -d. || true)
@@ -206,6 +206,36 @@ if [ -d "${CUDA_LIB_DIR}" ]; then
 
 fi
 
+if [ -z "${ONEAPI_ROOT}" ]; then
+    # Try the default location in case it exists
+    ONEAPI_ROOT=/opt/intel/oneapi
+fi
+
+if [ -z "${OLLAMA_SKIP_ONEAPI_GENERATE}" -a -d "${ONEAPI_ROOT}" ]; then
+    echo "OneAPI libraries detected - building dynamic OneAPI library"
+    init_vars
+    source ${ONEAPI_ROOT}/setvars.sh --force # set up environment variables for oneAPI
+    CC=icx
+    CMAKE_DEFS="${COMMON_CMAKE_DEFS} ${CMAKE_DEFS} -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DLLAMA_SYCL=ON -DLLAMA_SYCL_F16=OFF"
+    BUILD_DIR="../build/linux/${ARCH}/oneapi"
+    EXTRA_LIBS="-fsycl -Wl,-rpath,${ONEAPI_ROOT}/compiler/latest/lib,-rpath,${ONEAPI_ROOT}/mkl/latest/lib,-rpath,${ONEAPI_ROOT}/tbb/latest/lib,-rpath,${ONEAPI_ROOT}/compiler/latest/opt/oclfpga/linux64/lib -lOpenCL -lmkl_core -lmkl_sycl_blas -lmkl_intel_ilp64 -lmkl_tbb_thread -ltbb"
+    DEBUG_FLAGS="" # icx compiles with -O0 if we pass -g, so we must remove it
+    build
+
+    # copy oneAPI dependencies
+    for dep in $(ldd "${BUILD_DIR}/bin/ollama_llama_server" | grep "=>" | cut -f2 -d= | cut -f2 -d' ' | grep -e sycl -e mkl -e tbb); do
+        cp "${dep}" "${BUILD_DIR}/bin/"
+    done
+    cp "${ONEAPI_ROOT}/compiler/latest/lib/libOpenCL.so" "${BUILD_DIR}/bin/"
+    cp "${ONEAPI_ROOT}/compiler/latest/lib/libimf.so" "${BUILD_DIR}/bin/"
+    cp "${ONEAPI_ROOT}/compiler/latest/lib/libintlc.so.5" "${BUILD_DIR}/bin/"
+    cp "${ONEAPI_ROOT}/compiler/latest/lib/libirng.so" "${BUILD_DIR}/bin/"
+    cp "${ONEAPI_ROOT}/compiler/latest/lib/libpi_level_zero.so" "${BUILD_DIR}/bin/"
+    cp "${ONEAPI_ROOT}/compiler/latest/lib/libsvml.so" "${BUILD_DIR}/bin/"
+    cp "${ONEAPI_ROOT}/compiler/latest/lib/libur_loader.so.0" "${BUILD_DIR}/bin/"
+    compress
+fi
+
 if [ -z "${ROCM_PATH}" ]; then
     # Try the default location in case it exists
     ROCM_PATH=/opt/rocm
@@ -218,7 +248,7 @@ if [ -z "${CLBlast_DIR}" ]; then
     fi
 fi
 
-if [ -d "${ROCM_PATH}" ]; then
+if [ -z "${OLLAMA_SKIP_ROCM_GENERATE}" -a -d "${ROCM_PATH}" ]; then
     echo "ROCm libraries detected - building dynamic ROCm library"
     if [ -f ${ROCM_PATH}/lib/librocblas.so.*.*.????? ]; then
         ROCM_VARIANT=_v$(ls ${ROCM_PATH}/lib/librocblas.so.*.*.????? | cut -f5 -d. || true)

llm/generate/gen_windows.ps1

@@ -289,6 +289,49 @@ function build_cuda() {
     }
 }
 
+function build_oneapi() {
+  if ((-not "${env:OLLAMA_SKIP_ONEAPI_GENERATE}") -and ("${env:ONEAPI_ROOT}"))  {
+    # Get oneAPI version
+    $script:ONEAPI_VERSION = icpx --version
+    $script:ONEAPI_VERSION = [regex]::Match($script:ONEAPI_VERSION, '(?<=oneAPI DPC\+\+/C\+\+ Compiler )(?<version>\d+\.\d+\.\d+)').Value
+    if ($null -ne $script:ONEAPI_VERSION) {
+      $script:ONEAPI_VARIANT = "_v" + $script:ONEAPI_VERSION
+    }
+    init_vars
+    $script:buildDir = "../build/windows/${script:ARCH}/oneapi$script:ONEAPI_VARIANT"
+    $script:distDir ="$script:DIST_BASE\oneapi$script:ONEAPI_VARIANT"
+    $script:cmakeDefs += @(
+      "-G", "MinGW Makefiles",
+      "-DLLAMA_SYCL=ON",
+      "-DCMAKE_C_COMPILER=icx",
+      "-DCMAKE_CXX_COMPILER=icx",
+      "-DCMAKE_BUILD_TYPE=Release"
+    )
+
+    Write-Host "Building oneAPI"
+    build
+    # Ninja doesn't prefix with config name
+    if ($null -ne $script:DUMPBIN) {
+      & "$script:DUMPBIN" /dependents "${script:buildDir}/bin/ollama_llama_server.exe" | Select-String ".dll"
+    }
+    sign
+    install
+
+    cp "${env:ONEAPI_ROOT}\compiler\latest\bin\libirngmd.dll" "${script:distDir}"
+    cp "${env:ONEAPI_ROOT}\compiler\latest\bin\libmmd.dll" "${script:distDir}"
+    cp "${env:ONEAPI_ROOT}\compiler\latest\bin\pi_level_zero.dll" "${script:distDir}"
+    cp "${env:ONEAPI_ROOT}\compiler\latest\bin\pi_unified_runtime.dll" "${script:distDir}"
+    cp "${env:ONEAPI_ROOT}\compiler\latest\bin\pi_win_proxy_loader.dll" "${script:distDir}"
+    cp "${env:ONEAPI_ROOT}\compiler\latest\bin\svml_dispmd.dll" "${script:distDir}"
+    cp "${env:ONEAPI_ROOT}\compiler\latest\bin\sycl7.dll" "${script:distDir}"
+    cp "${env:ONEAPI_ROOT}\mkl\latest\bin\mkl_core.2.dll" "${script:distDir}"
+    cp "${env:ONEAPI_ROOT}\mkl\latest\bin\mkl_sycl_blas.4.dll" "${script:distDir}"
+    cp "${env:ONEAPI_ROOT}\mkl\latest\bin\mkl_tbb_thread.2.dll" "${script:distDir}"
+  } else {
+    Write-Host "Skipping oneAPI generation step"
+  }
+}
+
 function build_rocm() {
     if ((-not "${env:OLLAMA_SKIP_ROCM_GENERATE}") -and ("${env:HIP_PATH}")) {
         $script:ROCM_VERSION=(get-item $env:HIP_PATH).Basename
@@ -356,6 +399,7 @@ if ($($args.count) -eq 0) {
         build_cpu_avx
         build_cpu_avx2
         build_cuda
+        build_oneapi
         build_rocm
     }
 

llm/ggla.go

@@ -119,7 +119,7 @@ func (llm *ggla) decode(rs io.ReadSeeker) error {
 
 		t.Offset = uint64(offset)
 
-		if _, err := rs.Seek(int64(t.size()), io.SeekCurrent); err != nil {
+		if _, err := rs.Seek(int64(t.Size()), io.SeekCurrent); err != nil {
 			return err
 		}
 

llm/ggml.go

@@ -81,6 +81,11 @@ func (kv KV) ContextLength() uint64 {
 	return kv.u64(fmt.Sprintf("%s.context_length", kv.Architecture()))
 }
 
+func (kv KV) ChatTemplate() string {
+	s, _ := kv["tokenizer.chat_template"].(string)
+	return s
+}
+
 type Tensors []*Tensor
 
 func (ts Tensors) Layers() map[string]Layer {
@@ -106,7 +111,7 @@ type Layer map[string]*Tensor
 
 func (l Layer) size() (size uint64) {
 	for _, t := range l {
-		size += t.size()
+		size += t.Size()
 	}
 
 	return size
@@ -124,12 +129,12 @@ type Tensor struct {
 }
 
 func (t Tensor) blockSize() uint64 {
-	switch {
-	case t.Kind < 2:
+	switch t.Kind {
+	case 0, 1, 24, 25, 26, 27, 28, 30: // F32, F16, I8, I16, I32, I64, F64, BF16
 		return 1
-	case t.Kind < 10:
+	case 2, 3, 4, 5, 6, 7, 8, 9, 20: // Q4_0, Q4_1, Q5_0, Q5_1, Q8_0, Q8_1, IQ4_NL
 		return 32
-	default:
+	default: // All others
 		return 256
 	}
 }
@@ -171,7 +176,29 @@ func (t Tensor) typeSize() uint64 {
 	case 17: // IQ2_XS
 		return 2 + 2*blockSize/8 + blockSize/32
 	case 18: // IQ3_XXS
-		return 2 + 3*blockSize/8
+		return 2 + blockSize/4 + blockSize/8
+	case 19: // IQ1_S
+		return 2 + blockSize/8 + blockSize/16
+	case 20: // IQ4_NL
+		return 2 + blockSize/2
+	case 21: // IQ3_S
+		return 2 + blockSize/4 + blockSize/8 + blockSize/32 + 4
+	case 22: // IQ2_S
+		return 2 + blockSize/4 + blockSize/16
+	case 23: // IQ4_XS
+		return 2 + 2 + blockSize/2 + blockSize/64
+	case 24: // I8
+		return 1
+	case 25: // I16
+		return 2
+	case 26: // I32
+		return 4
+	case 27: // I64
+		return 8
+	case 28: // F64
+		return 8
+	case 29: // IQ1_M
+		return blockSize/8 + blockSize/16 + blockSize/32
 	default:
 		return 0
 	}
@@ -185,7 +212,7 @@ func (t Tensor) parameters() uint64 {
 	return count
 }
 
-func (t Tensor) size() uint64 {
+func (t Tensor) Size() uint64 {
 	return t.parameters() * t.typeSize() / t.blockSize()
 }
 
@@ -288,7 +315,7 @@ func (llm GGML) GraphSize(context, batch uint64) (partialOffload, fullOffload ui
 			// mixtral 8x22b
 			ff := uint64(llm.KV()["llama.feed_forward_length"].(uint32))
 			partialOffload = max(
-				3*ffnGateExpsWeight.size()+4*batch*(2*ff+headsKV+embedding+context+embedding/heads*headsKV),
+				3*ffnGateExpsWeight.Size()+4*batch*(2*ff+headsKV+embedding+context+embedding/heads*headsKV),
 				4*(context*batch*heads+context*embedding/heads*headsKV+batch*1024+embedding/heads*headsKV*batch),
 			)
 		} else if ffnGateWeight, ok := layers["blk.0"]["ffn_gate.0.weight"]; ok {

llm/gguf.go

@@ -62,16 +62,6 @@ func (c *containerGGUF) Decode(rs io.ReadSeeker) (model, error) {
 	return model, nil
 }
 
-const (
-	_ uint32 = iota
-	GGUFTokenNormal
-	GGUFTokenUnknown
-	GGUFTokenControl
-	GGUFTokenUserDefined
-	GGUFTokenUnused
-	GGUFTokenByte
-)
-
 const (
 	ggufTypeUint8 uint32 = iota
 	ggufTypeInt8
@@ -251,11 +241,11 @@ func (llm *gguf) Decode(rs io.ReadSeeker) error {
 	}
 
 	for _, tensor := range llm.tensors {
-		if _, err := rs.Seek(int64(tensor.size()), io.SeekCurrent); err != nil {
+		if _, err := rs.Seek(int64(tensor.Size()), io.SeekCurrent); err != nil {
 			return err
 		}
 
-		padding := llm.padding(int64(tensor.size()), int64(alignment))
+		padding := llm.padding(int64(tensor.Size()), int64(alignment))
 		if _, err := rs.Seek(padding, io.SeekCurrent); err != nil {
 			return err
 		}
@@ -480,9 +470,11 @@ var ggufKVOrder = map[string][]string{
 		"gemma.attention.key_length",
 		"gemma.attention.value_length",
 		"general.file_type",
+		"tokenizer.ggml.pre",
 		"tokenizer.ggml.model",
 		"tokenizer.ggml.tokens",
 		"tokenizer.ggml.scores",
+		"tokenizer.ggml.merges",
 		"tokenizer.ggml.token_type",
 		"tokenizer.ggml.bos_token_id",
 		"tokenizer.ggml.eos_token_id",
@@ -600,8 +592,8 @@ func (llm *gguf) Encode(ws io.WriteSeeker, kv KV, tensors []Tensor) error {
 			return err
 		}
 
-		dims := 0
-		for cnt := 0; cnt < len(tensor.Shape); cnt++ {
+		var dims int
+		for cnt := range len(tensor.Shape) {
 			if tensor.Shape[cnt] > 0 {
 				dims++
 			}
@@ -611,8 +603,8 @@ func (llm *gguf) Encode(ws io.WriteSeeker, kv KV, tensors []Tensor) error {
 			return err
 		}
 
-		for i := 0; i < dims; i++ {
-			if err := binary.Write(ws, llm.ByteOrder, uint64(tensor.Shape[dims-1-i])); err != nil {
+		for i := range dims {
+			if err := binary.Write(ws, llm.ByteOrder, tensor.Shape[dims-1-i]); err != nil {
 				return err
 			}
 		}
@@ -626,22 +618,8 @@ func (llm *gguf) Encode(ws io.WriteSeeker, kv KV, tensors []Tensor) error {
 		}
 	}
 
-	offset, err := ws.Seek(0, io.SeekCurrent)
-	if err != nil {
-		return err
-	}
-
 	var alignment int64 = 32
-	padding := llm.padding(offset, alignment)
-	if err := binary.Write(ws, llm.ByteOrder, bytes.Repeat([]byte{0}, int(padding))); err != nil {
-		return err
-	}
-
 	for _, tensor := range tensors {
-		if _, err := tensor.WriteTo(ws); err != nil {
-			return err
-		}
-
 		offset, err := ws.Seek(0, io.SeekCurrent)
 		if err != nil {
 			return err
@@ -651,6 +629,10 @@ func (llm *gguf) Encode(ws io.WriteSeeker, kv KV, tensors []Tensor) error {
 		if err := binary.Write(ws, llm.ByteOrder, bytes.Repeat([]byte{0}, int(padding))); err != nil {
 			return err
 		}
+
+		if _, err := tensor.WriteTo(ws); err != nil {
+			return err
+		}
 	}
 
 	return nil

llm/memory.go

@@ -5,9 +5,9 @@ import (
 	"log/slog"
 
 	"github.com/ollama/ollama/api"
+	"github.com/ollama/ollama/envconfig"
 	"github.com/ollama/ollama/format"
 	"github.com/ollama/ollama/gpu"
-	"github.com/ollama/ollama/server/envconfig"
 )
 
 // This algorithm looks for a complete fit to determine if we need to unload other models
@@ -53,6 +53,12 @@ func EstimateGPULayers(gpus []gpu.GpuInfo, ggml *GGML, projectors []string, opts
 		opts.NumCtx = max(opts.NumCtx, 2048)
 	}
 
+	layers := ggml.Tensors().Layers()
+	// add one layer worth of memory as a buffer
+	if blk0, ok := layers["blk.0"]; ok {
+		memoryMinimum += blk0.size()
+	}
+
 	// fp16 k,v = (1 (k) + 1 (v)) * sizeof(float16) * n_ctx * n_layer * n_embd / n_head * n_head_kv
 	var kv uint64 = 2 * 2 * uint64(opts.NumCtx) * ggml.KV().BlockCount() * ggml.KV().EmbeddingLength() / ggml.KV().HeadCount() * ggml.KV().HeadCountKV()
 
@@ -73,13 +79,11 @@ func EstimateGPULayers(gpus []gpu.GpuInfo, ggml *GGML, projectors []string, opts
 		graphPartialOffload = graphFullOffload
 	}
 
-	layers := ggml.Tensors().Layers()
-
 	// memoryRequiredTotal represents the memory required for full GPU offloading (all layers)
-	memoryRequiredTotal := memoryMinimum + graphFullOffload + layers["blk.0"].size()
+	memoryRequiredTotal := memoryMinimum + graphFullOffload
 
 	// memoryRequiredPartial represents the memory required for partial GPU offloading (n > 0, n < layers)
-	memoryRequiredPartial := memoryMinimum + graphPartialOffload + layers["blk.0"].size()
+	memoryRequiredPartial := memoryMinimum + graphPartialOffload
 
 	var memoryLayerOutput uint64
 	if layer, ok := layers["output_norm"]; ok {
@@ -99,16 +103,18 @@ func EstimateGPULayers(gpus []gpu.GpuInfo, ggml *GGML, projectors []string, opts
 	}
 
 	var layerCount int
-	for i := 0; i < int(ggml.KV().BlockCount()); i++ {
-		memoryLayer := layers[fmt.Sprintf("blk.%d", i)].size()
+	for i := range int(ggml.KV().BlockCount()) {
+		if blk, ok := layers[fmt.Sprintf("blk.%d", i)]; ok {
+			memoryLayer := blk.size()
 
-		// KV is proportional to the number of layers
-		memoryLayer += kv / ggml.KV().BlockCount()
+			// KV is proportional to the number of layers
+			memoryLayer += kv / ggml.KV().BlockCount()
 
-		memoryRequiredTotal += memoryLayer
-		if memoryAvailable > memoryRequiredPartial+memoryLayer {
-			memoryRequiredPartial += memoryLayer
-			layerCount++
+			memoryRequiredTotal += memoryLayer
+			if (opts.NumGPU >= 0 && layerCount+1 <= opts.NumGPU) || (opts.NumGPU < 0 && memoryAvailable > memoryRequiredPartial+memoryLayer) {
+				memoryRequiredPartial += memoryLayer
+				layerCount++
+			}
 		}
 	}
 
@@ -117,7 +123,7 @@ func EstimateGPULayers(gpus []gpu.GpuInfo, ggml *GGML, projectors []string, opts
 		memoryRequiredTotal += memoryLayerOutput
 	}
 
-	if memoryAvailable > memoryRequiredTotal {
+	if (opts.NumGPU >= 0 && layerCount+1 <= opts.NumGPU) || (opts.NumGPU < 0 && memoryAvailable > memoryRequiredTotal) {
 		layerCount = int(ggml.KV().BlockCount()) + 1
 		memoryRequiredPartial = memoryRequiredTotal
 	}
@@ -128,10 +134,10 @@ func EstimateGPULayers(gpus []gpu.GpuInfo, ggml *GGML, projectors []string, opts
 		"offload to gpu",
 		slog.Group(
 			"layers",
-			// actual number of layers offloaded
-			"real", opts.NumGPU,
+			// requested number of layers to offload
+			"requested", opts.NumGPU,
 			// estimated number of layers that can be offloaded
-			"estimate", layerCount,
+			"real", layerCount,
 		),
 		slog.Group(
 			"memory",

llm/patches/01-load-progress.diff

@@ -0,0 +1,31 @@
+diff --git a/common/common.cpp b/common/common.cpp
+index ba1ecf0e..cead57cc 100644
+--- a/common/common.cpp
++++ b/common/common.cpp
+@@ -1836,6 +1836,8 @@ struct llama_model_params llama_model_params_from_gpt_params(const gpt_params &
+     mparams.use_mmap        = params.use_mmap;
+     mparams.use_mlock       = params.use_mlock;
+     mparams.check_tensors   = params.check_tensors;
++    mparams.progress_callback = params.progress_callback;
++    mparams.progress_callback_user_data = params.progress_callback_user_data;
+     if (params.kv_overrides.empty()) {
+         mparams.kv_overrides = NULL;
+     } else {
+diff --git a/common/common.h b/common/common.h
+index d80344f2..71e84834 100644
+--- a/common/common.h
++++ b/common/common.h
+@@ -174,6 +174,13 @@ struct gpt_params {
+     // multimodal models (see examples/llava)
+     std::string mmproj = "";        // path to multimodal projector
+     std::vector<std::string> image; // path to image file(s)
++
++    // Called with a progress value between 0.0 and 1.0. Pass NULL to disable.
++    // If the provided progress_callback returns true, model loading continues.
++    // If it returns false, model loading is immediately aborted.
++    llama_progress_callback progress_callback = NULL;
++    // context pointer passed to the progress callback
++    void * progress_callback_user_data;
+ };
+ 
+ void gpt_params_handle_model_default(gpt_params & params);

llm/patches/03-load_exception.diff

@@ -1,8 +1,17 @@
+From 544a2d2e646d39e878d87dfbb3398a356bc560ab Mon Sep 17 00:00:00 2001
+From: Michael Yang <mxyng@pm.me>
+Date: Thu, 23 May 2024 11:18:45 -0700
+Subject: [PATCH] throw exception on load errors
+
+---
+ llama.cpp | 25 ++++++++++++++++---------
+ 1 file changed, 16 insertions(+), 9 deletions(-)
+
 diff --git a/llama.cpp b/llama.cpp
-index 4225f955..7b762f86 100644
+index 15c66077..8ba90b6a 100644
 --- a/llama.cpp
 +++ b/llama.cpp
-@@ -4756,7 +4756,7 @@ static int llama_model_load(const std::string & fname, llama_model & model, llam
+@@ -6346,7 +6346,7 @@ static int llama_model_load(const std::string & fname, llama_model & model, llam
          }
      } catch (const std::exception & err) {
          LLAMA_LOG_ERROR("%s: error loading model: %s\n", __func__, err.what());
@@ -11,10 +20,10 @@ index 4225f955..7b762f86 100644
      }
  
      return 0;
-@@ -12102,16 +12102,22 @@ struct llama_model * llama_load_model_from_file(
-         };
+@@ -15600,16 +15600,23 @@ struct llama_model * llama_load_model_from_file(
+         }
+         model->rpc_servers.push_back(servers);
      }
- 
 -    int status = llama_model_load(path_model, *model, params);
 -    GGML_ASSERT(status <= 0);
 -    if (status < 0) {
@@ -22,6 +31,7 @@ index 4225f955..7b762f86 100644
 -            LLAMA_LOG_ERROR("%s: failed to load model\n", __func__);
 -        } else if (status == -2) {
 -            LLAMA_LOG_INFO("%s: cancelled model load\n", __func__);
++
 +    try {
 +        int status = llama_model_load(path_model, *model, params);
 +        GGML_ASSERT(status <= 0);
@@ -42,3 +52,6 @@ index 4225f955..7b762f86 100644
      }
  
      return model;
+-- 
+2.45.1
+

llm/patches/05-clip-fix.diff

@@ -1,24 +0,0 @@
-diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp
-index e3c9bcd4..b43f892d 100644
---- a/examples/llava/clip.cpp
-+++ b/examples/llava/clip.cpp
-@@ -573,14 +573,16 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32
-     struct ggml_tensor * embeddings = inp;
-     if (ctx->has_class_embedding) {
-         embeddings = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, num_positions, batch_size);
-+    }
-+    ggml_set_name(embeddings, "embeddings");
-+    ggml_set_input(embeddings);
-+
-+    if (ctx->has_class_embedding) {
-         embeddings = ggml_acc(ctx0, embeddings, model.class_embedding,
-                 embeddings->nb[1], embeddings->nb[2], embeddings->nb[3], 0);
-         embeddings = ggml_acc(ctx0, embeddings, inp,
-                 embeddings->nb[1], embeddings->nb[2], embeddings->nb[3], model.class_embedding->nb[1]);
-     }
--    ggml_set_name(embeddings, "embeddings");
--    ggml_set_input(embeddings);
--
- 
-     struct ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_positions);
-     ggml_set_name(positions, "positions");

llm/patches/05-default-pretokenizer.diff

@@ -0,0 +1,32 @@
+diff --git a/llama.cpp b/llama.cpp
+index 40d2ec2c..74f3ee9c 100644
+--- a/llama.cpp
++++ b/llama.cpp
+@@ -4642,16 +4642,7 @@ static void llm_load_vocab(
+ 
+         // for now, only BPE models have pre-tokenizers
+         if (vocab.type == LLAMA_VOCAB_TYPE_BPE) {
+-            if (tokenizer_pre.empty()) {
+-                LLAMA_LOG_WARN("%s: missing pre-tokenizer type, using: 'default'\n", __func__);
+-                LLAMA_LOG_WARN("%s:                                             \n", __func__);
+-                LLAMA_LOG_WARN("%s: ************************************        \n", __func__);
+-                LLAMA_LOG_WARN("%s: GENERATION QUALITY WILL BE DEGRADED!        \n", __func__);
+-                LLAMA_LOG_WARN("%s: CONSIDER REGENERATING THE MODEL             \n", __func__);
+-                LLAMA_LOG_WARN("%s: ************************************        \n", __func__);
+-                LLAMA_LOG_WARN("%s:                                             \n", __func__);
+-                vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+-            } else if (
++            if (
+                     tokenizer_pre == "default") {
+                 vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+             } else if (
+@@ -4703,7 +4694,8 @@ static void llm_load_vocab(
+                 tokenizer_pre == "smaug-bpe") {
+                 vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_SMAUG;
+             } else {
+-                throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));
++                LLAMA_LOG_WARN("%s: missing or unrecognized pre-tokenizer type, using: 'default'\n", __func__);
++                vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+             }
+         } else {
+             vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;

llm/patches/06-qwen2.diff

@@ -0,0 +1,13 @@
+diff --git a/llama.cpp b/llama.cpp
+index 40d2ec2c..f34eb79a 100644
+--- a/llama.cpp
++++ b/llama.cpp
+@@ -6943,7 +6943,7 @@ static struct ggml_tensor * llm_build_kqv(
+         struct ggml_tensor * kq = ggml_mul_mat(ctx, k, q);
+         cb(kq, "kq", il);
+ 
+-        if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3 || model.arch == LLM_ARCH_GPTNEOX) {
++        if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3 || model.arch == LLM_ARCH_GPTNEOX || model.arch == LLM_ARCH_QWEN2) {
+             // for this arch, we need to perform the KQ multiplication with F32 precision, otherwise we get NaNs
+             // ref: https://github.com/ggerganov/llama.cpp/pull/4490#issuecomment-1859055847
+             ggml_mul_mat_set_prec(kq, GGML_PREC_F32);

llm/payload.go

@@ -10,9 +10,9 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"slices"
 	"strings"
 
-	"golang.org/x/exp/slices"
 	"golang.org/x/sync/errgroup"
 
 	"github.com/ollama/ollama/gpu"

llm/server.go

@@ -24,9 +24,9 @@ import (
 	"golang.org/x/sync/semaphore"
 
 	"github.com/ollama/ollama/api"
+	"github.com/ollama/ollama/envconfig"
 	"github.com/ollama/ollama/format"
 	"github.com/ollama/ollama/gpu"
-	"github.com/ollama/ollama/server/envconfig"
 )
 
 type LlamaServer interface {
@@ -38,6 +38,7 @@ type LlamaServer interface {
 	Detokenize(ctx context.Context, tokens []int) (string, error)
 	Close() error
 	EstimatedVRAM() uint64
+	EstimatedTotal() uint64
 }
 
 // llmServer is an instance of the llama.cpp server
@@ -54,6 +55,7 @@ type llmServer struct {
 	totalLayers    uint64
 	gpuCount       int
 	loadDuration   time.Duration // Record how long it took the model to load
+	loadProgress   float32
 
 	sem *semaphore.Weighted
 }
@@ -83,11 +85,11 @@ func NewLlamaServer(gpus gpu.GpuInfoList, model string, ggml *GGML, adapters, pr
 	var systemMemory uint64
 	gpuCount := len(gpus)
 	if (len(gpus) == 1 && gpus[0].Library == "cpu") || opts.NumGPU == 0 {
-
 		// TODO evaluate system memory to see if we should block the load, or force an unload of another CPU runner
 
 		cpuRunner = serverForCpu()
 		gpuCount = 0
+		_, _, estimatedTotal = EstimateGPULayers(gpus, ggml, projectors, opts)
 	} else {
 		if gpus[0].Library == "metal" {
 			memInfo, err := gpu.GetCPUMem()
@@ -101,21 +103,22 @@ func NewLlamaServer(gpus gpu.GpuInfoList, model string, ggml *GGML, adapters, pr
 		var layers int
 		layers, estimatedVRAM, estimatedTotal = EstimateGPULayers(gpus, ggml, projectors, opts)
 
-		if gpus[0].Library == "metal" && estimatedVRAM > systemMemory {
+		switch {
+		case gpus[0].Library == "metal" && estimatedVRAM > systemMemory:
 			// disable partial offloading when model is greater than total system memory as this
 			// can lead to locking up the system
 			opts.NumGPU = 0
-		} else if gpus[0].Library != "metal" && layers == 0 {
+		case gpus[0].Library != "metal" && layers == 0:
 			// Don't bother loading into the GPU if no layers can fit
 			cpuRunner = serverForCpu()
 			gpuCount = 0
-		} else if opts.NumGPU < 0 && layers > 0 && gpus[0].Library != "cpu" {
+		case opts.NumGPU < 0 && layers > 0 && gpus[0].Library != "cpu":
 			opts.NumGPU = layers
 		}
 	}
 
 	// Loop through potential servers
-	finalErr := fmt.Errorf("no suitable llama servers found")
+	finalErr := errors.New("no suitable llama servers found")
 
 	if len(adapters) > 1 {
 		return nil, errors.New("ollama supports only one lora adapter, but multiple were provided")
@@ -186,14 +189,34 @@ func NewLlamaServer(gpus gpu.GpuInfoList, model string, ggml *GGML, adapters, pr
 		params = append(params, "--memory-f32")
 	}
 
-	if opts.UseMLock {
-		params = append(params, "--mlock")
+	flashAttnEnabled := envconfig.FlashAttention
+
+	for _, g := range gpus {
+		// only cuda (compute capability 7+) and metal support flash attention
+		if g.Library != "metal" && (g.Library != "cuda" || g.DriverMajor < 7) {
+			flashAttnEnabled = false
+		}
+
+		// mmap has issues with partial offloading on metal
+		if g.Library == "metal" &&
+			uint64(opts.NumGPU) > 0 &&
+			uint64(opts.NumGPU) < ggml.KV().BlockCount()+1 {
+			opts.UseMMap = false
+		}
+	}
+
+	if flashAttnEnabled {
+		params = append(params, "--flash-attn")
 	}
 
 	if !opts.UseMMap {
 		params = append(params, "--no-mmap")
 	}
 
+	if opts.UseMLock {
+		params = append(params, "--mlock")
+	}
+
 	if opts.UseNUMA {
 		params = append(params, "--numa")
 	}
@@ -209,7 +232,7 @@ func NewLlamaServer(gpus gpu.GpuInfoList, model string, ggml *GGML, adapters, pr
 
 	params = append(params, "--parallel", fmt.Sprintf("%d", numParallel))
 
-	for i := 0; i < len(servers); i++ {
+	for i := range len(servers) {
 		dir := availableServers[servers[i]]
 		if dir == "" {
 			// Shouldn't happen
@@ -223,7 +246,7 @@ func NewLlamaServer(gpus gpu.GpuInfoList, model string, ggml *GGML, adapters, pr
 			gpuCount = 0
 		}
 
-		// Find an availableServers  port, retry on each iterration in case the failure was a port conflict race
+		// Find an availableServers  port, retry on each iteration in case the failure was a port conflict race
 		port := 0
 		if a, err := net.ResolveTCPAddr("tcp", "localhost:0"); err == nil {
 			var l *net.TCPListener
@@ -261,7 +284,7 @@ func NewLlamaServer(gpus gpu.GpuInfoList, model string, ggml *GGML, adapters, pr
 
 		server := filepath.Join(dir, "ollama_llama_server")
 		if runtime.GOOS == "windows" {
-			server = server + ".exe"
+			server += ".exe"
 		}
 
 		// Detect tmp cleaners wiping out the file
@@ -292,7 +315,7 @@ func NewLlamaServer(gpus gpu.GpuInfoList, model string, ggml *GGML, adapters, pr
 		s.cmd.Stdout = os.Stdout
 		s.cmd.Stderr = s.status
 
-		visibleDevicesEnv, visibleDevicesEnvVal := gpu.GpuInfoList(gpus).GetVisibleDevicesEnv()
+		visibleDevicesEnv, visibleDevicesEnvVal := gpus.GetVisibleDevicesEnv()
 		pathEnvVal := strings.Join(libraryPaths, string(filepath.ListSeparator))
 
 		// Update or add the path and visible devices variable with our adjusted version
@@ -316,8 +339,22 @@ func NewLlamaServer(gpus gpu.GpuInfoList, model string, ggml *GGML, adapters, pr
 		}
 
 		slog.Info("starting llama server", "cmd", s.cmd.String())
-		// Log at debug as the environment is inherited and might contain sensitive information
-		slog.Debug("subprocess", "environment", s.cmd.Env)
+		if envconfig.Debug {
+			filteredEnv := []string{}
+			for _, ev := range s.cmd.Env {
+				if strings.HasPrefix(ev, "CUDA_") ||
+					strings.HasPrefix(ev, "ROCM_") ||
+					strings.HasPrefix(ev, "HIP_") ||
+					strings.HasPrefix(ev, "HSA_") ||
+					strings.HasPrefix(ev, "GGML_") ||
+					strings.HasPrefix(ev, "PATH=") ||
+					strings.HasPrefix(ev, "LD_LIBRARY_PATH=") {
+					filteredEnv = append(filteredEnv, ev)
+				}
+			}
+			// Log at debug as the environment is inherited and might contain sensitive information
+			slog.Debug("subprocess", "environment", filteredEnv)
+		}
 
 		if err = s.cmd.Start(); err != nil {
 			// Detect permission denied and augment them essage about noexec
@@ -392,10 +429,11 @@ func (s ServerStatus) ToString() string {
 }
 
 type ServerStatusResp struct {
-	Status          string `json:"status"`
-	SlotsIdle       int    `json:"slots_idle"`
-	SlotsProcessing int    `json:"slots_processing"`
-	Error           string `json:"error"`
+	Status          string  `json:"status"`
+	SlotsIdle       int     `json:"slots_idle"`
+	SlotsProcessing int     `json:"slots_processing"`
+	Error           string  `json:"error"`
+	Progress        float32 `json:"progress"`
 }
 
 func (s *llmServer) getServerStatus(ctx context.Context) (ServerStatus, error) {
@@ -421,7 +459,7 @@ func (s *llmServer) getServerStatus(ctx context.Context) (ServerStatus, error) {
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		if errors.Is(err, context.DeadlineExceeded) {
-			return ServerStatusNotResponding, fmt.Errorf("server not responding")
+			return ServerStatusNotResponding, errors.New("server not responding")
 		}
 		return ServerStatusError, fmt.Errorf("health resp: %w", err)
 	}
@@ -443,6 +481,7 @@ func (s *llmServer) getServerStatus(ctx context.Context) (ServerStatus, error) {
 	case "no slot available":
 		return ServerStatusNoSlotsAvailable, nil
 	case "loading model":
+		s.loadProgress = status.Progress
 		return ServerStatusLoadingModel, nil
 	default:
 		return ServerStatusError, fmt.Errorf("server error: %+v", status)
@@ -483,15 +522,18 @@ func (s *llmServer) Ping(ctx context.Context) error {
 
 func (s *llmServer) WaitUntilRunning(ctx context.Context) error {
 	start := time.Now()
-	expiresAt := time.Now().Add(10 * time.Minute) // be generous with timeout, large models can take a while to load
+	stallDuration := 5 * time.Minute            // If no progress happens
+	finalLoadDuration := 5 * time.Minute        // After we hit 100%, give the runner more time to come online
+	stallTimer := time.Now().Add(stallDuration) // give up if we stall
 
 	slog.Info("waiting for llama runner to start responding")
 	var lastStatus ServerStatus = -1
+	fullyLoaded := false
 
 	for {
 		select {
 		case <-ctx.Done():
-			slog.Info("context expired before server started")
+			slog.Warn("client connection closed before server finished loading, aborting load")
 			return fmt.Errorf("timed out waiting for llama runner to start: %w", ctx.Err())
 		case err := <-s.done:
 			msg := ""
@@ -501,13 +543,13 @@ func (s *llmServer) WaitUntilRunning(ctx context.Context) error {
 			return fmt.Errorf("llama runner process has terminated: %v %s", err, msg)
 		default:
 		}
-		if time.Now().After(expiresAt) {
+		if time.Now().After(stallTimer) {
 			// timeout
 			msg := ""
 			if s.status != nil && s.status.LastErrMsg != "" {
 				msg = s.status.LastErrMsg
 			}
-			return fmt.Errorf("timed out waiting for llama runner to start: %s", msg)
+			return fmt.Errorf("timed out waiting for llama runner to start - progress %0.2f - %s", s.loadProgress, msg)
 		}
 		if s.cmd.ProcessState != nil {
 			msg := ""
@@ -518,6 +560,7 @@ func (s *llmServer) WaitUntilRunning(ctx context.Context) error {
 		}
 		ctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond)
 		defer cancel()
+		priorProgress := s.loadProgress
 		status, _ := s.getServerStatus(ctx)
 		if lastStatus != status && status != ServerStatusReady {
 			// Only log on status changes
@@ -530,6 +573,15 @@ func (s *llmServer) WaitUntilRunning(ctx context.Context) error {
 			return nil
 		default:
 			lastStatus = status
+			// Reset the timer as long as we're making forward progress on the load
+			if priorProgress != s.loadProgress {
+				slog.Debug(fmt.Sprintf("model load progress %0.2f", s.loadProgress))
+				stallTimer = time.Now().Add(stallDuration)
+			} else if !fullyLoaded && int(s.loadProgress*100.0) >= 100 {
+				slog.Debug("model load completed, waiting for server to become available", "status", status.ToString())
+				stallTimer = time.Now().Add(finalLoadDuration)
+				fullyLoaded = true
+			}
 			time.Sleep(time.Millisecond * 250)
 			continue
 		}
@@ -554,7 +606,7 @@ array  ::=
 
 string ::=
   "\"" (
-    [^"\\] |
+    [^"\\\x7F\x00-\x1F] |
     "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes
   )* "\"" ws
 
@@ -713,7 +765,7 @@ func (s *llmServer) Completion(ctx context.Context, req CompletionRequest, fn fu
 
 			var c completion
 			if err := json.Unmarshal(evt, &c); err != nil {
-				return fmt.Errorf("error unmarshaling llm prediction response: %v", err)
+				return fmt.Errorf("error unmarshalling llm prediction response: %v", err)
 			}
 
 			switch {
@@ -955,6 +1007,10 @@ func (s *llmServer) EstimatedVRAM() uint64 {
 	return s.estimatedVRAM
 }
 
+func (s *llmServer) EstimatedTotal() uint64 {
+	return s.estimatedTotal
+}
+
 func parseDurationMs(ms float64) time.Duration {
 	dur, err := time.ParseDuration(fmt.Sprintf("%fms", ms))
 	if err != nil {

macapp/src/index.ts

@@ -162,7 +162,7 @@ app.on('before-quit', () => {
   }
 })
 
-const updateURL = `https://ollama.ai/api/update?os=${process.platform}&arch=${
+const updateURL = `https://ollama.com/api/update?os=${process.platform}&arch=${
   process.arch
 }&version=${app.getVersion()}&id=${id()}`
 

openai/openai.go

@@ -211,6 +211,29 @@ func fromRequest(r ChatCompletionRequest) api.ChatRequest {
 	}
 }
 
+type DeleteCompletion struct {
+	Id      string `json:"id"`
+	Object  string `json:"object"`
+	Deleted bool   `json:"deleted"`
+}
+
+func toDeleteCompletion(model string) DeleteCompletion {
+	return DeleteCompletion{
+		Id:      model,
+		Object:  "model",
+		Deleted: true,
+	}
+}
+
+type BaseWriter struct {
+	gin.ResponseWriter
+}
+
+type DeleteWriter struct {
+	BaseWriter
+	model string
+}
+
 type writer struct {
 	stream bool
 	id     string
@@ -245,7 +268,6 @@ func (w *writer) writeResponse(data []byte) (int, error) {
 		d, err := json.Marshal(toChunk(w.id, chatResponse))
 		if err != nil {
 			return 0, err
-
 		}
 
 		w.ResponseWriter.Header().Set("Content-Type", "text/event-stream")
@@ -316,3 +338,69 @@ func Middleware() gin.HandlerFunc {
 		c.Next()
 	}
 }
+
+func (w *DeleteWriter) writeError(code int, data []byte) (int, error) {
+	var serr api.StatusError
+	err := json.Unmarshal(data, &serr)
+	if err != nil {
+		return 0, err
+	}
+
+	w.ResponseWriter.Header().Set("Content-Type", "application/json")
+	err = json.NewEncoder(w.ResponseWriter).Encode(NewError(http.StatusInternalServerError, serr.Error()))
+	if err != nil {
+		return 0, err
+	}
+
+	return len(data), nil
+}
+
+func (w *DeleteWriter) writeResponse(data []byte) (int, error) {
+	// delete completion
+	w.ResponseWriter.Header().Set("Content-Type", "application/json")
+	err := json.NewEncoder(w.ResponseWriter).Encode(toDeleteCompletion(w.model))
+	if err != nil {
+		return 0, err
+	}
+
+	return len(data), nil
+}
+
+func (w *DeleteWriter) Write(data []byte) (int, error) {
+	code := w.ResponseWriter.Status()
+	if code != http.StatusOK {
+		return w.writeError(code, data)
+	}
+
+	return w.writeResponse(data)
+}
+
+func DeleteMiddleware() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		var b bytes.Buffer
+		if err := json.NewEncoder(&b).Encode(api.DeleteRequest{Model: c.Param("model")}); err != nil {
+			c.AbortWithStatusJSON(http.StatusInternalServerError, NewError(http.StatusInternalServerError, err.Error()))
+			return
+		}
+
+		c.Request.Body = io.NopCloser(&b)
+
+		// response writer
+		w := &DeleteWriter{
+			BaseWriter: BaseWriter{ResponseWriter: c.Writer},
+			model:      c.Param("model"),
+		}
+
+		c.Writer = w
+
+		c.Next()
+
+		// If the status code is OK, write the DeleteCompletion response
+		if c.Writer.Status() == http.StatusOK {
+			_, err := w.writeResponse(nil)
+			if err != nil {
+				c.AbortWithStatusJSON(http.StatusInternalServerError, NewError(http.StatusInternalServerError, err.Error()))
+			}
+		}
+	}
+}

parser/parser.go

@@ -1,4 +1,4 @@
-package model
+package parser
 
 import (
 	"bufio"
@@ -8,6 +8,9 @@ import (
 	"io"
 	"strconv"
 	"strings"
+
+	"golang.org/x/text/encoding/unicode"
+	"golang.org/x/text/transform"
 )
 
 type File struct {
@@ -70,7 +73,9 @@ func ParseFile(r io.Reader) (*File, error) {
 
 	var f File
 
-	br := bufio.NewReader(r)
+	tr := unicode.BOMOverride(unicode.UTF8.NewDecoder())
+	br := bufio.NewReader(transform.NewReader(r, tr))
+
 	for {
 		r, _, err := br.ReadRune()
 		if errors.Is(err, io.EOF) {

parser/parser_test.go

@@ -1,13 +1,18 @@
-package model
+package parser
 
 import (
 	"bytes"
+	"encoding/binary"
 	"fmt"
 	"io"
 	"strings"
 	"testing"
+	"unicode/utf16"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/encoding/unicode"
 )
 
 func TestParseFileFile(t *testing.T) {
@@ -23,7 +28,7 @@ TEMPLATE template1
 	reader := strings.NewReader(input)
 
 	modelfile, err := ParseFile(reader)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 
 	expectedCommands := []Command{
 		{Name: "model", Args: "model1"},
@@ -86,7 +91,7 @@ func TestParseFileFrom(t *testing.T) {
 	for _, c := range cases {
 		t.Run("", func(t *testing.T) {
 			modelfile, err := ParseFile(strings.NewReader(c.input))
-			assert.ErrorIs(t, err, c.err)
+			require.ErrorIs(t, err, c.err)
 			if modelfile != nil {
 				assert.Equal(t, c.expected, modelfile.Commands)
 			}
@@ -103,7 +108,7 @@ PARAMETER param1
 	reader := strings.NewReader(input)
 
 	_, err := ParseFile(reader)
-	assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
+	require.ErrorIs(t, err, io.ErrUnexpectedEOF)
 }
 
 func TestParseFileBadCommand(t *testing.T) {
@@ -112,8 +117,7 @@ FROM foo
 BADCOMMAND param1 value1
 `
 	_, err := ParseFile(strings.NewReader(input))
-	assert.ErrorIs(t, err, errInvalidCommand)
-
+	require.ErrorIs(t, err, errInvalidCommand)
 }
 
 func TestParseFileMessages(t *testing.T) {
@@ -199,7 +203,7 @@ MESSAGE system`,
 	for _, c := range cases {
 		t.Run("", func(t *testing.T) {
 			modelfile, err := ParseFile(strings.NewReader(c.input))
-			assert.ErrorIs(t, err, c.err)
+			require.ErrorIs(t, err, c.err)
 			if modelfile != nil {
 				assert.Equal(t, c.expected, modelfile.Commands)
 			}
@@ -353,7 +357,7 @@ TEMPLATE """
 	for _, c := range cases {
 		t.Run("", func(t *testing.T) {
 			modelfile, err := ParseFile(strings.NewReader(c.multiline))
-			assert.ErrorIs(t, err, c.err)
+			require.ErrorIs(t, err, c.err)
 			if modelfile != nil {
 				assert.Equal(t, c.expected, modelfile.Commands)
 			}
@@ -411,7 +415,7 @@ func TestParseFileParameters(t *testing.T) {
 			fmt.Fprintln(&b, "FROM foo")
 			fmt.Fprintln(&b, "PARAMETER", k)
 			modelfile, err := ParseFile(&b)
-			assert.NoError(t, err)
+			require.NoError(t, err)
 
 			assert.Equal(t, []Command{
 				{Name: "model", Args: "foo"},
@@ -440,7 +444,7 @@ FROM foo
 	for _, c := range cases {
 		t.Run("", func(t *testing.T) {
 			modelfile, err := ParseFile(strings.NewReader(c.input))
-			assert.NoError(t, err)
+			require.NoError(t, err)
 			assert.Equal(t, c.expected, modelfile.Commands)
 		})
 	}
@@ -499,13 +503,76 @@ SYSTEM ""
 	for _, c := range cases {
 		t.Run("", func(t *testing.T) {
 			modelfile, err := ParseFile(strings.NewReader(c))
-			assert.NoError(t, err)
+			require.NoError(t, err)
 
 			modelfile2, err := ParseFile(strings.NewReader(modelfile.String()))
-			assert.NoError(t, err)
+			require.NoError(t, err)
 
 			assert.Equal(t, modelfile, modelfile2)
 		})
 	}
-
+}
+
+func TestParseFileUTF16ParseFile(t *testing.T) {
+	data := `FROM bob
+PARAMETER param1 1
+PARAMETER param2 4096
+SYSTEM You are a utf16 file.
+`
+
+	expected := []Command{
+		{Name: "model", Args: "bob"},
+		{Name: "param1", Args: "1"},
+		{Name: "param2", Args: "4096"},
+		{Name: "system", Args: "You are a utf16 file."},
+	}
+
+	t.Run("le", func(t *testing.T) {
+		var b bytes.Buffer
+		require.NoError(t, binary.Write(&b, binary.LittleEndian, []byte{0xff, 0xfe}))
+		require.NoError(t, binary.Write(&b, binary.LittleEndian, utf16.Encode([]rune(data))))
+
+		actual, err := ParseFile(&b)
+		require.NoError(t, err)
+
+		assert.Equal(t, expected, actual.Commands)
+	})
+
+	t.Run("be", func(t *testing.T) {
+		var b bytes.Buffer
+		require.NoError(t, binary.Write(&b, binary.BigEndian, []byte{0xfe, 0xff}))
+		require.NoError(t, binary.Write(&b, binary.BigEndian, utf16.Encode([]rune(data))))
+
+		actual, err := ParseFile(&b)
+		require.NoError(t, err)
+		assert.Equal(t, expected, actual.Commands)
+	})
+}
+
+func TestParseMultiByte(t *testing.T) {
+	input := `FROM test
+	SYSTEM 你好👋`
+
+	expect := []Command{
+		{Name: "model", Args: "test"},
+		{Name: "system", Args: "你好👋"},
+	}
+
+	encodings := []encoding.Encoding{
+		unicode.UTF8,
+		unicode.UTF16(unicode.LittleEndian, unicode.UseBOM),
+		unicode.UTF16(unicode.BigEndian, unicode.UseBOM),
+	}
+
+	for _, encoding := range encodings {
+		t.Run(fmt.Sprintf("%s", encoding), func(t *testing.T) {
+			s, err := encoding.NewEncoder().String(input)
+			require.NoError(t, err)
+
+			actual, err := ParseFile(strings.NewReader(s))
+			require.NoError(t, err)
+
+			assert.Equal(t, expect, actual.Commands)
+		})
+	}
 }

progress/progress.go

@@ -59,7 +59,7 @@ func (p *Progress) StopAndClear() bool {
 	stopped := p.stop()
 	if stopped {
 		// clear all progress lines
-		for i := 0; i < p.pos; i++ {
+		for i := range p.pos {
 			if i > 0 {
 				fmt.Fprint(p.w, "\033[A")
 			}
@@ -85,7 +85,7 @@ func (p *Progress) render() {
 	defer fmt.Fprint(p.w, "\033[?25h")
 
 	// clear already rendered progress lines
-	for i := 0; i < p.pos; i++ {
+	for i := range p.pos {
 		if i > 0 {
 			fmt.Fprint(p.w, "\033[A")
 		}

readline/buffer.go

@@ -5,16 +5,20 @@ import (
 	"os"
 
 	"github.com/emirpasic/gods/lists/arraylist"
+	"github.com/mattn/go-runewidth"
 	"golang.org/x/term"
 )
 
 type Buffer struct {
-	Pos       int
-	Buf       *arraylist.List
-	Prompt    *Prompt
-	LineWidth int
-	Width     int
-	Height    int
+	DisplayPos int
+	Pos        int
+	Buf        *arraylist.List
+	//LineHasSpace is an arraylist of bools to keep track of whether a line has a space at the end
+	LineHasSpace *arraylist.List
+	Prompt       *Prompt
+	LineWidth    int
+	Width        int
+	Height       int
 }
 
 func NewBuffer(prompt *Prompt) (*Buffer, error) {
@@ -27,25 +31,56 @@ func NewBuffer(prompt *Prompt) (*Buffer, error) {
 	lwidth := width - len(prompt.prompt())
 
 	b := &Buffer{
-		Pos:       0,
-		Buf:       arraylist.New(),
-		Prompt:    prompt,
-		Width:     width,
-		Height:    height,
-		LineWidth: lwidth,
+		DisplayPos:   0,
+		Pos:          0,
+		Buf:          arraylist.New(),
+		LineHasSpace: arraylist.New(),
+		Prompt:       prompt,
+		Width:        width,
+		Height:       height,
+		LineWidth:    lwidth,
 	}
 
 	return b, nil
 }
 
+func (b *Buffer) GetLineSpacing(line int) bool {
+	hasSpace, _ := b.LineHasSpace.Get(line)
+
+	if hasSpace == nil {
+		return false
+	}
+
+	return hasSpace.(bool)
+}
+
 func (b *Buffer) MoveLeft() {
 	if b.Pos > 0 {
-		if b.Pos%b.LineWidth == 0 {
-			fmt.Printf(CursorUp + CursorBOL + cursorRightN(b.Width))
-		} else {
-			fmt.Print(CursorLeft)
+		//asserts that we retrieve a rune
+		if e, ok := b.Buf.Get(b.Pos - 1); ok {
+			if r, ok := e.(rune); ok {
+				rLength := runewidth.RuneWidth(r)
+
+				if b.DisplayPos%b.LineWidth == 0 {
+					fmt.Printf(CursorUp + CursorBOL + cursorRightN(b.Width))
+					if rLength == 2 {
+						fmt.Print(CursorLeft)
+					}
+
+					line := b.DisplayPos/b.LineWidth - 1
+					hasSpace := b.GetLineSpacing(line)
+					if hasSpace {
+						b.DisplayPos -= 1
+						fmt.Print(CursorLeft)
+					}
+				} else {
+					fmt.Print(cursorLeftN(rLength))
+				}
+
+				b.Pos -= 1
+				b.DisplayPos -= rLength
+			}
 		}
-		b.Pos -= 1
 	}
 }
 
@@ -71,18 +106,32 @@ func (b *Buffer) MoveLeftWord() {
 }
 
 func (b *Buffer) MoveRight() {
-	if b.Pos < b.Size() {
-		b.Pos += 1
-		if b.Pos%b.LineWidth == 0 {
-			fmt.Printf(CursorDown + CursorBOL + cursorRightN(len(b.Prompt.prompt())))
-		} else {
-			fmt.Print(CursorRight)
+	if b.Pos < b.Buf.Size() {
+		if e, ok := b.Buf.Get(b.Pos); ok {
+			if r, ok := e.(rune); ok {
+				rLength := runewidth.RuneWidth(r)
+				b.Pos += 1
+				hasSpace := b.GetLineSpacing(b.DisplayPos / b.LineWidth)
+				b.DisplayPos += rLength
+
+				if b.DisplayPos%b.LineWidth == 0 {
+					fmt.Printf(CursorDown + CursorBOL + cursorRightN(len(b.Prompt.prompt())))
+				} else if (b.DisplayPos-rLength)%b.LineWidth == b.LineWidth-1 && hasSpace {
+					fmt.Printf(CursorDown + CursorBOL + cursorRightN(len(b.Prompt.prompt())+rLength))
+					b.DisplayPos += 1
+				} else if b.LineHasSpace.Size() > 0 && b.DisplayPos%b.LineWidth == b.LineWidth-1 && hasSpace {
+					fmt.Printf(CursorDown + CursorBOL + cursorRightN(len(b.Prompt.prompt())))
+					b.DisplayPos += 1
+				} else {
+					fmt.Print(cursorRightN(rLength))
+				}
+			}
 		}
 	}
 }
 
 func (b *Buffer) MoveRightWord() {
-	if b.Pos < b.Size() {
+	if b.Pos < b.Buf.Size() {
 		for {
 			b.MoveRight()
 			v, _ := b.Buf.Get(b.Pos)
@@ -90,7 +139,7 @@ func (b *Buffer) MoveRightWord() {
 				break
 			}
 
-			if b.Pos == b.Size() {
+			if b.Pos == b.Buf.Size() {
 				break
 			}
 		}
@@ -99,89 +148,200 @@ func (b *Buffer) MoveRightWord() {
 
 func (b *Buffer) MoveToStart() {
 	if b.Pos > 0 {
-		currLine := b.Pos / b.LineWidth
+		currLine := b.DisplayPos / b.LineWidth
 		if currLine > 0 {
-			for cnt := 0; cnt < currLine; cnt++ {
+			for range currLine {
 				fmt.Print(CursorUp)
 			}
 		}
 		fmt.Printf(CursorBOL + cursorRightN(len(b.Prompt.prompt())))
 		b.Pos = 0
+		b.DisplayPos = 0
 	}
 }
 
 func (b *Buffer) MoveToEnd() {
-	if b.Pos < b.Size() {
-		currLine := b.Pos / b.LineWidth
-		totalLines := b.Size() / b.LineWidth
+	if b.Pos < b.Buf.Size() {
+		currLine := b.DisplayPos / b.LineWidth
+		totalLines := b.DisplaySize() / b.LineWidth
 		if currLine < totalLines {
-			for cnt := 0; cnt < totalLines-currLine; cnt++ {
+			for range totalLines - currLine {
 				fmt.Print(CursorDown)
 			}
-			remainder := b.Size() % b.LineWidth
+			remainder := b.DisplaySize() % b.LineWidth
 			fmt.Printf(CursorBOL + cursorRightN(len(b.Prompt.prompt())+remainder))
 		} else {
-			fmt.Print(cursorRightN(b.Size() - b.Pos))
+			fmt.Print(cursorRightN(b.DisplaySize() - b.DisplayPos))
 		}
 
-		b.Pos = b.Size()
+		b.Pos = b.Buf.Size()
+		b.DisplayPos = b.DisplaySize()
 	}
 }
 
-func (b *Buffer) Size() int {
-	return b.Buf.Size()
+func (b *Buffer) DisplaySize() int {
+	sum := 0
+	for i := range b.Buf.Size() {
+		if e, ok := b.Buf.Get(i); ok {
+			if r, ok := e.(rune); ok {
+				sum += runewidth.RuneWidth(r)
+			}
+		}
+	}
+
+	return sum
 }
 
 func (b *Buffer) Add(r rune) {
 	if b.Pos == b.Buf.Size() {
-		fmt.Printf("%c", r)
-		b.Buf.Add(r)
-		b.Pos += 1
-		if b.Pos > 0 && b.Pos%b.LineWidth == 0 {
+		b.AddChar(r, false)
+	} else {
+		b.AddChar(r, true)
+	}
+}
+
+func (b *Buffer) AddChar(r rune, insert bool) {
+	rLength := runewidth.RuneWidth(r)
+	b.DisplayPos += rLength
+
+	if b.Pos > 0 {
+		if b.DisplayPos%b.LineWidth == 0 {
+			fmt.Printf("%c", r)
 			fmt.Printf("\n%s", b.Prompt.AltPrompt)
+
+			if insert {
+				b.LineHasSpace.Set(b.DisplayPos/b.LineWidth-1, false)
+			} else {
+				b.LineHasSpace.Add(false)
+			}
+
+			// this case occurs when a double-width rune crosses the line boundary
+		} else if b.DisplayPos%b.LineWidth < (b.DisplayPos-rLength)%b.LineWidth {
+			if insert {
+				fmt.Print(ClearToEOL)
+			}
+			fmt.Printf("\n%s", b.Prompt.AltPrompt)
+			b.DisplayPos += 1
+			fmt.Printf("%c", r)
+
+			if insert {
+				b.LineHasSpace.Set(b.DisplayPos/b.LineWidth-1, true)
+			} else {
+				b.LineHasSpace.Add(true)
+			}
+		} else {
+			fmt.Printf("%c", r)
 		}
 	} else {
 		fmt.Printf("%c", r)
+	}
+
+	if insert {
 		b.Buf.Insert(b.Pos, r)
-		b.Pos += 1
-		if b.Pos > 0 && b.Pos%b.LineWidth == 0 {
-			fmt.Printf("\n%s", b.Prompt.AltPrompt)
-		}
+	} else {
+		b.Buf.Add(r)
+	}
+
+	b.Pos += 1
+
+	if insert {
 		b.drawRemaining()
 	}
 }
 
+func (b *Buffer) countRemainingLineWidth(place int) int {
+	var sum int
+	counter := -1
+	var prevLen int
+
+	for place <= b.LineWidth {
+		counter += 1
+		sum += prevLen
+		if e, ok := b.Buf.Get(b.Pos + counter); ok {
+			if r, ok := e.(rune); ok {
+				place += runewidth.RuneWidth(r)
+				prevLen = len(string(r))
+			}
+		} else {
+			break
+		}
+	}
+
+	return sum
+}
+
 func (b *Buffer) drawRemaining() {
 	var place int
 	remainingText := b.StringN(b.Pos)
 	if b.Pos > 0 {
-		place = b.Pos % b.LineWidth
+		place = b.DisplayPos % b.LineWidth
 	}
 	fmt.Print(CursorHide)
 
 	// render the rest of the current line
-	currLine := remainingText[:min(b.LineWidth-place, len(remainingText))]
+	currLineLength := b.countRemainingLineWidth(place)
+
+	currLine := remainingText[:min(currLineLength, len(remainingText))]
+	currLineSpace := runewidth.StringWidth(currLine)
+	remLength := runewidth.StringWidth(remainingText)
+
 	if len(currLine) > 0 {
 		fmt.Printf(ClearToEOL + currLine)
-		fmt.Print(cursorLeftN(len(currLine)))
+		fmt.Print(cursorLeftN(currLineSpace))
 	} else {
 		fmt.Print(ClearToEOL)
 	}
 
+	if currLineSpace != b.LineWidth-place && currLineSpace != remLength {
+		b.LineHasSpace.Set(b.DisplayPos/b.LineWidth, true)
+	} else if currLineSpace != b.LineWidth-place {
+		b.LineHasSpace.Remove(b.DisplayPos / b.LineWidth)
+	} else {
+		b.LineHasSpace.Set(b.DisplayPos/b.LineWidth, false)
+	}
+
+	if (b.DisplayPos+currLineSpace)%b.LineWidth == 0 && currLine == remainingText {
+		fmt.Print(cursorRightN(currLineSpace))
+		fmt.Printf("\n%s", b.Prompt.AltPrompt)
+		fmt.Printf(CursorUp + CursorBOL + cursorRightN(b.Width-currLineSpace))
+	}
+
 	// render the other lines
-	if len(remainingText) > len(currLine) {
-		remaining := []rune(remainingText[len(currLine):])
+	if remLength > currLineSpace {
+		remaining := (remainingText[len(currLine):])
 		var totalLines int
-		for i, c := range remaining {
-			if i%b.LineWidth == 0 {
+		var displayLength int
+		var lineLength int = currLineSpace
+
+		for _, c := range remaining {
+			if displayLength == 0 || (displayLength+runewidth.RuneWidth(c))%b.LineWidth < displayLength%b.LineWidth {
 				fmt.Printf("\n%s", b.Prompt.AltPrompt)
 				totalLines += 1
+
+				if displayLength != 0 {
+					if lineLength == b.LineWidth {
+						b.LineHasSpace.Set(b.DisplayPos/b.LineWidth+totalLines-1, false)
+					} else {
+						b.LineHasSpace.Set(b.DisplayPos/b.LineWidth+totalLines-1, true)
+					}
+				}
+
+				lineLength = 0
 			}
+
+			displayLength += runewidth.RuneWidth(c)
+			lineLength += runewidth.RuneWidth(c)
 			fmt.Printf("%c", c)
 		}
 		fmt.Print(ClearToEOL)
 		fmt.Print(cursorUpN(totalLines))
-		fmt.Printf(CursorBOL + cursorRightN(b.Width-len(currLine)))
+		fmt.Printf(CursorBOL + cursorRightN(b.Width-currLineSpace))
+
+		hasSpace := b.GetLineSpacing(b.DisplayPos / b.LineWidth)
+
+		if hasSpace && b.DisplayPos%b.LineWidth != b.LineWidth-1 {
+			fmt.Print(CursorLeft)
+		}
 	}
 
 	fmt.Print(CursorShow)
@@ -189,46 +349,81 @@ func (b *Buffer) drawRemaining() {
 
 func (b *Buffer) Remove() {
 	if b.Buf.Size() > 0 && b.Pos > 0 {
-		if b.Pos%b.LineWidth == 0 {
-			// if the user backspaces over the word boundary, do this magic to clear the line
-			// and move to the end of the previous line
-			fmt.Printf(CursorBOL + ClearToEOL)
-			fmt.Printf(CursorUp + CursorBOL + cursorRightN(b.Width) + " " + CursorLeft)
-		} else {
-			fmt.Printf(CursorLeft + " " + CursorLeft)
-		}
+		if e, ok := b.Buf.Get(b.Pos - 1); ok {
+			if r, ok := e.(rune); ok {
+				rLength := runewidth.RuneWidth(r)
+				hasSpace := b.GetLineSpacing(b.DisplayPos/b.LineWidth - 1)
 
-		var eraseExtraLine bool
-		if (b.Size()-1)%b.LineWidth == 0 {
-			eraseExtraLine = true
-		}
+				if b.DisplayPos%b.LineWidth == 0 {
+					// if the user backspaces over the word boundary, do this magic to clear the line
+					// and move to the end of the previous line
+					fmt.Printf(CursorBOL + ClearToEOL)
+					fmt.Printf(CursorUp + CursorBOL + cursorRightN(b.Width))
 
-		b.Pos -= 1
-		b.Buf.Remove(b.Pos)
+					if b.DisplaySize()%b.LineWidth < (b.DisplaySize()-rLength)%b.LineWidth {
+						b.LineHasSpace.Remove(b.DisplayPos/b.LineWidth - 1)
+					}
 
-		if b.Pos < b.Size() {
-			b.drawRemaining()
-			// this erases a line which is left over when backspacing in the middle of a line and there
-			// are trailing characters which go over the line width boundary
-			if eraseExtraLine {
-				remainingLines := (b.Size() - b.Pos) / b.LineWidth
-				fmt.Printf(cursorDownN(remainingLines+1) + CursorBOL + ClearToEOL)
-				place := b.Pos % b.LineWidth
-				fmt.Printf(cursorUpN(remainingLines+1) + cursorRightN(place+len(b.Prompt.prompt())))
+					if hasSpace {
+						b.DisplayPos -= 1
+						fmt.Print(CursorLeft)
+					}
+
+					if rLength == 2 {
+						fmt.Print(CursorLeft + "  " + cursorLeftN(2))
+					} else {
+						fmt.Print(" " + CursorLeft)
+					}
+				} else if (b.DisplayPos-rLength)%b.LineWidth == 0 && hasSpace {
+					fmt.Printf(CursorBOL + ClearToEOL)
+					fmt.Printf(CursorUp + CursorBOL + cursorRightN(b.Width))
+
+					if b.Pos == b.Buf.Size() {
+						b.LineHasSpace.Remove(b.DisplayPos/b.LineWidth - 1)
+					}
+					b.DisplayPos -= 1
+				} else {
+					fmt.Print(cursorLeftN(rLength))
+					for range rLength {
+						fmt.Print(" ")
+					}
+					fmt.Print(cursorLeftN(rLength))
+				}
+
+				var eraseExtraLine bool
+				if (b.DisplaySize()-1)%b.LineWidth == 0 || (rLength == 2 && ((b.DisplaySize()-2)%b.LineWidth == 0)) || b.DisplaySize()%b.LineWidth == 0 {
+					eraseExtraLine = true
+				}
+
+				b.Pos -= 1
+				b.DisplayPos -= rLength
+				b.Buf.Remove(b.Pos)
+
+				if b.Pos < b.Buf.Size() {
+					b.drawRemaining()
+					// this erases a line which is left over when backspacing in the middle of a line and there
+					// are trailing characters which go over the line width boundary
+					if eraseExtraLine {
+						remainingLines := (b.DisplaySize() - b.DisplayPos) / b.LineWidth
+						fmt.Printf(cursorDownN(remainingLines+1) + CursorBOL + ClearToEOL)
+						place := b.DisplayPos % b.LineWidth
+						fmt.Printf(cursorUpN(remainingLines+1) + cursorRightN(place+len(b.Prompt.prompt())))
+					}
+				}
 			}
 		}
 	}
 }
 
 func (b *Buffer) Delete() {
-	if b.Size() > 0 && b.Pos < b.Size() {
+	if b.Buf.Size() > 0 && b.Pos < b.Buf.Size() {
 		b.Buf.Remove(b.Pos)
 		b.drawRemaining()
-		if b.Size()%b.LineWidth == 0 {
-			if b.Pos != b.Size() {
-				remainingLines := (b.Size() - b.Pos) / b.LineWidth
+		if b.DisplaySize()%b.LineWidth == 0 {
+			if b.DisplayPos != b.DisplaySize() {
+				remainingLines := (b.DisplaySize() - b.DisplayPos) / b.LineWidth
 				fmt.Printf(cursorDownN(remainingLines) + CursorBOL + ClearToEOL)
-				place := b.Pos % b.LineWidth
+				place := b.DisplayPos % b.LineWidth
 				fmt.Printf(cursorUpN(remainingLines) + cursorRightN(place+len(b.Prompt.prompt())))
 			}
 		}
@@ -244,9 +439,9 @@ func (b *Buffer) DeleteBefore() {
 }
 
 func (b *Buffer) DeleteRemaining() {
-	if b.Size() > 0 && b.Pos < b.Size() {
-		charsToDel := b.Size() - b.Pos
-		for cnt := 0; cnt < charsToDel; cnt++ {
+	if b.DisplaySize() > 0 && b.Pos < b.DisplaySize() {
+		charsToDel := b.Buf.Size() - b.Pos
+		for range charsToDel {
 			b.Delete()
 		}
 	}
@@ -281,14 +476,16 @@ func (b *Buffer) ClearScreen() {
 		ph := b.Prompt.placeholder()
 		fmt.Printf(ColorGrey + ph + cursorLeftN(len(ph)) + ColorDefault)
 	} else {
-		currPos := b.Pos
+		currPos := b.DisplayPos
+		currIndex := b.Pos
 		b.Pos = 0
+		b.DisplayPos = 0
 		b.drawRemaining()
 		fmt.Printf(CursorReset + cursorRightN(len(b.Prompt.prompt())))
 		if currPos > 0 {
 			targetLine := currPos / b.LineWidth
 			if targetLine > 0 {
-				for cnt := 0; cnt < targetLine; cnt++ {
+				for range targetLine {
 					fmt.Print(CursorDown)
 				}
 			}
@@ -300,7 +497,8 @@ func (b *Buffer) ClearScreen() {
 				fmt.Printf(CursorBOL + b.Prompt.AltPrompt)
 			}
 		}
-		b.Pos = currPos
+		b.Pos = currIndex
+		b.DisplayPos = currPos
 	}
 }
 
@@ -309,9 +507,20 @@ func (b *Buffer) IsEmpty() bool {
 }
 
 func (b *Buffer) Replace(r []rune) {
+	b.DisplayPos = 0
 	b.Pos = 0
+	lineNums := b.DisplaySize() / b.LineWidth
+
 	b.Buf.Clear()
-	fmt.Printf(ClearLine + CursorBOL + b.Prompt.prompt())
+
+	fmt.Printf(CursorBOL + ClearToEOL)
+
+	for range lineNums {
+		fmt.Print(CursorUp + CursorBOL + ClearToEOL)
+	}
+
+	fmt.Printf(CursorBOL + b.Prompt.prompt())
+
 	for _, c := range r {
 		b.Add(c)
 	}
@@ -328,7 +537,7 @@ func (b *Buffer) StringN(n int) string {
 func (b *Buffer) StringNM(n, m int) string {
 	var s string
 	if m == 0 {
-		m = b.Size()
+		m = b.Buf.Size()
 	}
 	for cnt := n; cnt < m; cnt++ {
 		c, _ := b.Buf.Get(cnt)

readline/history.go

@@ -91,7 +91,7 @@ func (h *History) Add(l []rune) {
 func (h *History) Compact() {
 	s := h.Buf.Size()
 	if s > h.Limit {
-		for cnt := 0; cnt < s-h.Limit; cnt++ {
+		for range s - h.Limit {
 			h.Buf.Remove(0)
 		}
 	}
@@ -139,7 +139,7 @@ func (h *History) Save() error {
 	defer f.Close()
 
 	buf := bufio.NewWriter(f)
-	for cnt := 0; cnt < h.Size(); cnt++ {
+	for cnt := range h.Size() {
 		v, _ := h.Buf.Get(cnt)
 		line, _ := v.([]rune)
 		if _, err := buf.WriteString(string(line) + "\n"); err != nil {

readline/readline.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"syscall"
 )
 
 type Prompt struct {
@@ -63,7 +62,7 @@ func New(prompt Prompt) (*Instance, error) {
 
 func (i *Instance) Readline() (string, error) {
 	if !i.Terminal.rawmode {
-		fd := int(syscall.Stdin)
+		fd := os.Stdin.Fd()
 		termios, err := SetRawMode(fd)
 		if err != nil {
 			return "", err
@@ -80,8 +79,8 @@ func (i *Instance) Readline() (string, error) {
 	fmt.Print(prompt)
 
 	defer func() {
-		fd := int(syscall.Stdin)
-		// nolint: errcheck
+		fd := os.Stdin.Fd()
+		//nolint:errcheck
 		UnsetRawMode(fd, i.Terminal.termios)
 		i.Terminal.rawmode = false
 	}()
@@ -136,7 +135,7 @@ func (i *Instance) Readline() (string, error) {
 				buf.MoveRight()
 			case CharBracketedPaste:
 				var code string
-				for cnt := 0; cnt < 3; cnt++ {
+				for range 3 {
 					r, err = i.Terminal.Read()
 					if err != nil {
 						return "", io.EOF
@@ -150,7 +149,7 @@ func (i *Instance) Readline() (string, error) {
 					i.Pasting = false
 				}
 			case KeyDel:
-				if buf.Size() > 0 {
+				if buf.DisplaySize() > 0 {
 					buf.Delete()
 				}
 				metaDel = true
@@ -198,11 +197,11 @@ func (i *Instance) Readline() (string, error) {
 			buf.Remove()
 		case CharTab:
 			// todo: convert back to real tabs
-			for cnt := 0; cnt < 8; cnt++ {
+			for range 8 {
 				buf.Add(' ')
 			}
 		case CharDelete:
-			if buf.Size() > 0 {
+			if buf.DisplaySize() > 0 {
 				buf.Delete()
 			} else {
 				return "", io.EOF
@@ -216,7 +215,7 @@ func (i *Instance) Readline() (string, error) {
 		case CharCtrlW:
 			buf.DeleteWord()
 		case CharCtrlZ:
-			fd := int(syscall.Stdin)
+			fd := os.Stdin.Fd()
 			return handleCharCtrlZ(fd, i.Terminal.termios)
 		case CharEnter, CharCtrlJ:
 			output := buf.String()
@@ -248,7 +247,7 @@ func (i *Instance) HistoryDisable() {
 }
 
 func NewTerminal() (*Terminal, error) {
-	fd := int(syscall.Stdin)
+	fd := os.Stdin.Fd()
 	termios, err := SetRawMode(fd)
 	if err != nil {
 		return nil, err

readline/readline_unix.go

@@ -6,7 +6,7 @@ import (
 	"syscall"
 )
 
-func handleCharCtrlZ(fd int, termios any) (string, error) {
+func handleCharCtrlZ(fd uintptr, termios any) (string, error) {
 	t := termios.(*Termios)
 	if err := UnsetRawMode(fd, t); err != nil {
 		return "", err

readline/readline_windows.go

@@ -1,6 +1,6 @@
 package readline
 
-func handleCharCtrlZ(fd int, state any) (string, error) {
+func handleCharCtrlZ(fd uintptr, state any) (string, error) {
 	// not supported
 	return "", nil
 }

readline/term.go

@@ -8,7 +8,7 @@ import (
 
 type Termios syscall.Termios
 
-func SetRawMode(fd int) (*Termios, error) {
+func SetRawMode(fd uintptr) (*Termios, error) {
 	termios, err := getTermios(fd)
 	if err != nil {
 		return nil, err
@@ -25,13 +25,13 @@ func SetRawMode(fd int) (*Termios, error) {
 	return termios, setTermios(fd, &newTermios)
 }
 
-func UnsetRawMode(fd int, termios any) error {
+func UnsetRawMode(fd uintptr, termios any) error {
 	t := termios.(*Termios)
 	return setTermios(fd, t)
 }
 
 // IsTerminal returns true if the given file descriptor is a terminal.
-func IsTerminal(fd int) bool {
+func IsTerminal(fd uintptr) bool {
 	_, err := getTermios(fd)
 	return err == nil
 }

readline/term_bsd.go

@@ -7,17 +7,17 @@ import (
 	"unsafe"
 )
 
-func getTermios(fd int) (*Termios, error) {
+func getTermios(fd uintptr) (*Termios, error) {
 	termios := new(Termios)
-	_, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), syscall.TIOCGETA, uintptr(unsafe.Pointer(termios)), 0, 0, 0)
+	_, _, err := syscall.Syscall6(syscall.SYS_IOCTL, fd, syscall.TIOCGETA, uintptr(unsafe.Pointer(termios)), 0, 0, 0)
 	if err != 0 {
 		return nil, err
 	}
 	return termios, nil
 }
 
-func setTermios(fd int, termios *Termios) error {
-	_, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), syscall.TIOCSETA, uintptr(unsafe.Pointer(termios)), 0, 0, 0)
+func setTermios(fd uintptr, termios *Termios) error {
+	_, _, err := syscall.Syscall6(syscall.SYS_IOCTL, fd, syscall.TIOCSETA, uintptr(unsafe.Pointer(termios)), 0, 0, 0)
 	if err != 0 {
 		return err
 	}

readline/term_linux.go

@@ -10,17 +10,17 @@ import (
 const tcgets = 0x5401
 const tcsets = 0x5402
 
-func getTermios(fd int) (*Termios, error) {
+func getTermios(fd uintptr) (*Termios, error) {
 	termios := new(Termios)
-	_, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), tcgets, uintptr(unsafe.Pointer(termios)), 0, 0, 0)
+	_, _, err := syscall.Syscall6(syscall.SYS_IOCTL, fd, tcgets, uintptr(unsafe.Pointer(termios)), 0, 0, 0)
 	if err != 0 {
 		return nil, err
 	}
 	return termios, nil
 }
 
-func setTermios(fd int, termios *Termios) error {
-	_, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), tcsets, uintptr(unsafe.Pointer(termios)), 0, 0, 0)
+func setTermios(fd uintptr, termios *Termios) error {
+	_, _, err := syscall.Syscall6(syscall.SYS_IOCTL, fd, tcsets, uintptr(unsafe.Pointer(termios)), 0, 0, 0)
 	if err != 0 {
 		return err
 	}

readline/term_windows.go

@@ -9,13 +9,13 @@ type State struct {
 }
 
 // IsTerminal checks if the given file descriptor is associated with a terminal
-func IsTerminal(fd int) bool {
+func IsTerminal(fd uintptr) bool {
 	var st uint32
 	err := windows.GetConsoleMode(windows.Handle(fd), &st)
 	return err == nil
 }
 
-func SetRawMode(fd int) (*State, error) {
+func SetRawMode(fd uintptr) (*State, error) {
 	var st uint32
 	if err := windows.GetConsoleMode(windows.Handle(fd), &st); err != nil {
 		return nil, err
@@ -32,7 +32,7 @@ func SetRawMode(fd int) (*State, error) {
 	return &State{st}, nil
 }
 
-func UnsetRawMode(fd int, state any) error {
+func UnsetRawMode(fd uintptr, state any) error {
 	s := state.(*State)
 	return windows.SetConsoleMode(windows.Handle(fd), s.mode)
 }

scripts/install.sh

@@ -33,9 +33,11 @@ case "$ARCH" in
     *) error "Unsupported architecture: $ARCH" ;;
 esac
 
+IS_WSL2=false
+
 KERN=$(uname -r)
 case "$KERN" in
-    *icrosoft*WSL2 | *icrosoft*wsl2) ;;
+    *icrosoft*WSL2 | *icrosoft*wsl2) IS_WSL2=true;;
     *icrosoft) error "Microsoft WSL1 is not currently supported. Please upgrade to WSL2 with 'wsl --set-version <distro> 2'" ;;
     *) ;;
 esac
@@ -72,7 +74,7 @@ status "Installing ollama to $BINDIR..."
 $SUDO install -o0 -g0 -m755 -d $BINDIR
 $SUDO install -o0 -g0 -m755 $TEMP_DIR/ollama $BINDIR/ollama
 
-install_success() { 
+install_success() {
     status 'The Ollama API is now available at 127.0.0.1:11434.'
     status 'Install complete. Run "ollama" from the command line.'
 }
@@ -131,6 +133,17 @@ if available systemctl; then
     configure_systemd
 fi
 
+# WSL2 only supports GPUs via nvidia passthrough
+# so check for nvidia-smi to determine if GPU is available
+if [ "$IS_WSL2" = true ]; then
+    if available nvidia-smi && [ -n "$(nvidia-smi | grep -o "CUDA Version: [0-9]*\.[0-9]*")" ]; then
+        status "Nvidia GPU detected."
+    fi
+    install_success
+    exit 0
+fi
+
+# Install GPU dependencies on Linux
 if ! available lspci && ! available lshw; then
     warning "Unable to detect NVIDIA/AMD GPU. Install lspci or lshw to automatically detect and install GPU dependencies."
     exit 0
@@ -139,12 +152,12 @@ fi
 check_gpu() {
     # Look for devices based on vendor ID for NVIDIA and AMD
     case $1 in
-        lspci) 
+        lspci)
             case $2 in
                 nvidia) available lspci && lspci -d '10de:' | grep -q 'NVIDIA' || return 1 ;;
                 amdgpu) available lspci && lspci -d '1002:' | grep -q 'AMD' || return 1 ;;
             esac ;;
-        lshw) 
+        lshw)
             case $2 in
                 nvidia) available lshw && $SUDO lshw -c display -numeric | grep -q 'vendor: .* \[10DE\]' || return 1 ;;
                 amdgpu) available lshw && $SUDO lshw -c display -numeric | grep -q 'vendor: .* \[1002\]' || return 1 ;;
@@ -181,7 +194,7 @@ if check_gpu lspci amdgpu || check_gpu lshw amdgpu; then
     curl --fail --show-error --location --progress-bar "https://ollama.com/download/ollama-linux-amd64-rocm.tgz${VER_PARAM}" \
         | $SUDO tar zx --owner ollama --group ollama -C /usr/share/ollama/lib/rocm .
     install_success
-    status "AMD GPU dependencies installed."
+    status "AMD GPU ready."
     exit 0
 fi
 
@@ -274,7 +287,7 @@ if ! check_gpu nvidia-smi || [ -z "$(nvidia-smi | grep -o "CUDA Version: [0-9]*\
     esac
 fi
 
-if ! lsmod | grep -q nvidia; then
+if ! lsmod | grep -q nvidia || ! lsmod | grep -q nvidia_uvm; then
     KERNEL_RELEASE="$(uname -r)"
     case $OS_NAME in
         rocky) $SUDO $PACKAGE_MANAGER -y install kernel-devel kernel-headers ;;
@@ -295,7 +308,19 @@ if ! lsmod | grep -q nvidia; then
     fi
 
     $SUDO modprobe nvidia
+    $SUDO modprobe nvidia_uvm
 fi
 
+# make sure the NVIDIA modules are loaded on boot with nvidia-persistenced
+if command -v nvidia-persistenced > /dev/null 2>&1; then
+    $SUDO touch /etc/modules-load.d/nvidia.conf
+    MODULES="nvidia nvidia-uvm"
+    for MODULE in $MODULES; do
+        if ! grep -qxF "$MODULE" /etc/modules-load.d/nvidia.conf; then
+            echo "$MODULE" | sudo tee -a /etc/modules-load.d/nvidia.conf > /dev/null
+        fi
+    done
+fi
 
-status "NVIDIA CUDA drivers installed."
+status "NVIDIA GPU ready."
+install_success

server/download.go

@@ -221,7 +221,7 @@ func (b *blobDownload) downloadChunk(ctx context.Context, requestURL *url.URL, w
 		}
 		defer resp.Body.Close()
 
-		n, err := io.Copy(w, io.TeeReader(resp.Body, part))
+		n, err := io.CopyN(w, io.TeeReader(resp.Body, part), part.Size-part.Completed)
 		if err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, io.ErrUnexpectedEOF) {
 			// rollback progress
 			b.Completed.Add(-n)
@@ -340,17 +340,17 @@ type downloadOpts struct {
 }
 
 // downloadBlob downloads a blob from the registry and stores it in the blobs directory
-func downloadBlob(ctx context.Context, opts downloadOpts) error {
+func downloadBlob(ctx context.Context, opts downloadOpts) (cacheHit bool, _ error) {
 	fp, err := GetBlobsPath(opts.digest)
 	if err != nil {
-		return err
+		return false, err
 	}
 
 	fi, err := os.Stat(fp)
 	switch {
 	case errors.Is(err, os.ErrNotExist):
 	case err != nil:
-		return err
+		return false, err
 	default:
 		opts.fn(api.ProgressResponse{
 			Status:    fmt.Sprintf("pulling %s", opts.digest[7:19]),
@@ -359,7 +359,7 @@ func downloadBlob(ctx context.Context, opts downloadOpts) error {
 			Completed: fi.Size(),
 		})
 
-		return nil
+		return true, nil
 	}
 
 	data, ok := blobDownloadManager.LoadOrStore(opts.digest, &blobDownload{Name: fp, Digest: opts.digest})
@@ -369,12 +369,12 @@ func downloadBlob(ctx context.Context, opts downloadOpts) error {
 		requestURL = requestURL.JoinPath("v2", opts.mp.GetNamespaceRepository(), "blobs", opts.digest)
 		if err := download.Prepare(ctx, requestURL, opts.regOpts); err != nil {
 			blobDownloadManager.Delete(opts.digest)
-			return err
+			return false, err
 		}
 
-		// nolint: contextcheck
+		//nolint:contextcheck
 		go download.Run(context.Background(), requestURL, opts.regOpts)
 	}
 
-	return download.Wait(ctx, opts.fn)
+	return false, download.Wait(ctx, opts.fn)
 }

server/envconfig/config.go

@@ -1,174 +0,0 @@
-package envconfig
-
-import (
-	"fmt"
-	"log/slog"
-	"os"
-	"path/filepath"
-	"runtime"
-	"strconv"
-	"strings"
-)
-
-var (
-	// Set via OLLAMA_ORIGINS in the environment
-	AllowOrigins []string
-	// Set via OLLAMA_DEBUG in the environment
-	Debug bool
-	// Set via OLLAMA_LLM_LIBRARY in the environment
-	LLMLibrary string
-	// Set via OLLAMA_MAX_LOADED_MODELS in the environment
-	MaxRunners int
-	// Set via OLLAMA_MAX_QUEUE in the environment
-	MaxQueuedRequests int
-	// Set via OLLAMA_MAX_VRAM in the environment
-	MaxVRAM uint64
-	// Set via OLLAMA_NOPRUNE in the environment
-	NoPrune bool
-	// Set via OLLAMA_NUM_PARALLEL in the environment
-	NumParallel int
-	// Set via OLLAMA_RUNNERS_DIR in the environment
-	RunnersDir string
-	// Set via OLLAMA_TMPDIR in the environment
-	TmpDir string
-)
-
-func AsMap() map[string]string {
-	return map[string]string{
-		"OLLAMA_ORIGINS":           fmt.Sprintf("%v", AllowOrigins),
-		"OLLAMA_DEBUG":             fmt.Sprintf("%v", Debug),
-		"OLLAMA_LLM_LIBRARY":       fmt.Sprintf("%v", LLMLibrary),
-		"OLLAMA_MAX_LOADED_MODELS": fmt.Sprintf("%v", MaxRunners),
-		"OLLAMA_MAX_QUEUE":         fmt.Sprintf("%v", MaxQueuedRequests),
-		"OLLAMA_MAX_VRAM":          fmt.Sprintf("%v", MaxVRAM),
-		"OLLAMA_NOPRUNE":           fmt.Sprintf("%v", NoPrune),
-		"OLLAMA_NUM_PARALLEL":      fmt.Sprintf("%v", NumParallel),
-		"OLLAMA_RUNNERS_DIR":       fmt.Sprintf("%v", RunnersDir),
-		"OLLAMA_TMPDIR":            fmt.Sprintf("%v", TmpDir),
-	}
-}
-
-var defaultAllowOrigins = []string{
-	"localhost",
-	"127.0.0.1",
-	"0.0.0.0",
-}
-
-// Clean quotes and spaces from the value
-func clean(key string) string {
-	return strings.Trim(os.Getenv(key), "\"' ")
-}
-
-func init() {
-	// default values
-	NumParallel = 1
-	MaxRunners = 1
-	MaxQueuedRequests = 512
-
-	LoadConfig()
-}
-
-func LoadConfig() {
-	if debug := clean("OLLAMA_DEBUG"); debug != "" {
-		d, err := strconv.ParseBool(debug)
-		if err == nil {
-			Debug = d
-		} else {
-			Debug = true
-		}
-	}
-
-	RunnersDir = clean("OLLAMA_RUNNERS_DIR")
-	if runtime.GOOS == "windows" && RunnersDir == "" {
-		// On Windows we do not carry the payloads inside the main executable
-		appExe, err := os.Executable()
-		if err != nil {
-			slog.Error("failed to lookup executable path", "error", err)
-		}
-
-		cwd, err := os.Getwd()
-		if err != nil {
-			slog.Error("failed to lookup working directory", "error", err)
-		}
-
-		var paths []string
-		for _, root := range []string{filepath.Dir(appExe), cwd} {
-			paths = append(paths,
-				filepath.Join(root),
-				filepath.Join(root, "windows-"+runtime.GOARCH),
-				filepath.Join(root, "dist", "windows-"+runtime.GOARCH),
-			)
-		}
-
-		// Try a few variations to improve developer experience when building from source in the local tree
-		for _, p := range paths {
-			candidate := filepath.Join(p, "ollama_runners")
-			_, err := os.Stat(candidate)
-			if err == nil {
-				RunnersDir = candidate
-				break
-			}
-		}
-		if RunnersDir == "" {
-			slog.Error("unable to locate llm runner directory.  Set OLLAMA_RUNNERS_DIR to the location of 'ollama_runners'")
-		}
-	}
-
-	TmpDir = clean("OLLAMA_TMPDIR")
-
-	userLimit := clean("OLLAMA_MAX_VRAM")
-	if userLimit != "" {
-		avail, err := strconv.ParseUint(userLimit, 10, 64)
-		if err != nil {
-			slog.Error("invalid setting, ignoring", "OLLAMA_MAX_VRAM", userLimit, "error", err)
-		} else {
-			MaxVRAM = avail
-		}
-	}
-
-	LLMLibrary = clean("OLLAMA_LLM_LIBRARY")
-
-	if onp := clean("OLLAMA_NUM_PARALLEL"); onp != "" {
-		val, err := strconv.Atoi(onp)
-		if err != nil || val <= 0 {
-			slog.Error("invalid setting must be greater than zero", "OLLAMA_NUM_PARALLEL", onp, "error", err)
-		} else {
-			NumParallel = val
-		}
-	}
-
-	if noprune := clean("OLLAMA_NOPRUNE"); noprune != "" {
-		NoPrune = true
-	}
-
-	if origins := clean("OLLAMA_ORIGINS"); origins != "" {
-		AllowOrigins = strings.Split(origins, ",")
-	}
-	for _, allowOrigin := range defaultAllowOrigins {
-		AllowOrigins = append(AllowOrigins,
-			fmt.Sprintf("http://%s", allowOrigin),
-			fmt.Sprintf("https://%s", allowOrigin),
-			fmt.Sprintf("http://%s:*", allowOrigin),
-			fmt.Sprintf("https://%s:*", allowOrigin),
-		)
-	}
-
-	maxRunners := clean("OLLAMA_MAX_LOADED_MODELS")
-	if maxRunners != "" {
-		m, err := strconv.Atoi(maxRunners)
-		if err != nil {
-			slog.Error("invalid setting", "OLLAMA_MAX_LOADED_MODELS", maxRunners, "error", err)
-		} else {
-			MaxRunners = m
-		}
-	}
-
-	if onp := os.Getenv("OLLAMA_MAX_QUEUE"); onp != "" {
-		p, err := strconv.Atoi(onp)
-		if err != nil || p <= 0 {
-			slog.Error("invalid setting", "OLLAMA_MAX_QUEUE", onp, "error", err)
-		} else {
-			MaxQueuedRequests = p
-		}
-	}
-}

server/envconfig/config_test.go

@@ -1,20 +0,0 @@
-package envconfig
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestConfig(t *testing.T) {
-	Debug = false // Reset whatever was loaded in init()
-	t.Setenv("OLLAMA_DEBUG", "")
-	LoadConfig()
-	require.False(t, Debug)
-	t.Setenv("OLLAMA_DEBUG", "false")
-	LoadConfig()
-	require.False(t, Debug)
-	t.Setenv("OLLAMA_DEBUG", "1")
-	LoadConfig()
-	require.True(t, Debug)
-}

server/images.go

@@ -18,16 +18,16 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"slices"
 	"strconv"
 	"strings"
 
-	"golang.org/x/exp/slices"
-
 	"github.com/ollama/ollama/api"
 	"github.com/ollama/ollama/auth"
+	"github.com/ollama/ollama/envconfig"
 	"github.com/ollama/ollama/format"
 	"github.com/ollama/ollama/llm"
-	"github.com/ollama/ollama/server/envconfig"
+	"github.com/ollama/ollama/parser"
 	"github.com/ollama/ollama/types/errtypes"
 	"github.com/ollama/ollama/types/model"
 	"github.com/ollama/ollama/version"
@@ -61,36 +61,36 @@ func (m *Model) IsEmbedding() bool {
 }
 
 func (m *Model) String() string {
-	var modelfile model.File
+	var modelfile parser.File
 
-	modelfile.Commands = append(modelfile.Commands, model.Command{
+	modelfile.Commands = append(modelfile.Commands, parser.Command{
 		Name: "model",
 		Args: m.ModelPath,
 	})
 
 	for _, adapter := range m.AdapterPaths {
-		modelfile.Commands = append(modelfile.Commands, model.Command{
+		modelfile.Commands = append(modelfile.Commands, parser.Command{
 			Name: "adapter",
 			Args: adapter,
 		})
 	}
 
 	for _, projector := range m.ProjectorPaths {
-		modelfile.Commands = append(modelfile.Commands, model.Command{
+		modelfile.Commands = append(modelfile.Commands, parser.Command{
 			Name: "model",
 			Args: projector,
 		})
 	}
 
 	if m.Template != "" {
-		modelfile.Commands = append(modelfile.Commands, model.Command{
+		modelfile.Commands = append(modelfile.Commands, parser.Command{
 			Name: "template",
 			Args: m.Template,
 		})
 	}
 
 	if m.System != "" {
-		modelfile.Commands = append(modelfile.Commands, model.Command{
+		modelfile.Commands = append(modelfile.Commands, parser.Command{
 			Name: "system",
 			Args: m.System,
 		})
@@ -100,13 +100,13 @@ func (m *Model) String() string {
 		switch v := v.(type) {
 		case []any:
 			for _, s := range v {
-				modelfile.Commands = append(modelfile.Commands, model.Command{
+				modelfile.Commands = append(modelfile.Commands, parser.Command{
 					Name: k,
 					Args: fmt.Sprintf("%v", s),
 				})
 			}
 		default:
-			modelfile.Commands = append(modelfile.Commands, model.Command{
+			modelfile.Commands = append(modelfile.Commands, parser.Command{
 				Name: k,
 				Args: fmt.Sprintf("%v", v),
 			})
@@ -114,14 +114,14 @@ func (m *Model) String() string {
 	}
 
 	for _, license := range m.License {
-		modelfile.Commands = append(modelfile.Commands, model.Command{
+		modelfile.Commands = append(modelfile.Commands, parser.Command{
 			Name: "license",
 			Args: license,
 		})
 	}
 
 	for _, msg := range m.Messages {
-		modelfile.Commands = append(modelfile.Commands, model.Command{
+		modelfile.Commands = append(modelfile.Commands, parser.Command{
 			Name: "message",
 			Args: fmt.Sprintf("%s %s", msg.Role, msg.Content),
 		})
@@ -314,7 +314,7 @@ func realpath(rel, from string) string {
 	return abspath
 }
 
-func CreateModel(ctx context.Context, name, modelFileDir, quantization string, modelfile *model.File, fn func(resp api.ProgressResponse)) (err error) {
+func CreateModel(ctx context.Context, name model.Name, modelFileDir, quantization string, modelfile *parser.File, fn func(resp api.ProgressResponse)) (err error) {
 	config := ConfigV2{
 		OS:           "linux",
 		Architecture: "amd64",
@@ -332,14 +332,31 @@ func CreateModel(ctx context.Context, name, modelFileDir, quantization string, m
 
 		switch c.Name {
 		case "model", "adapter":
-			var baseLayers []*layerWithGGML
+			var baseLayers []*layerGGML
 			if name := model.ParseName(c.Args); name.IsValid() {
 				baseLayers, err = parseFromModel(ctx, name, fn)
 				if err != nil {
 					return err
 				}
 			} else if strings.HasPrefix(c.Args, "@") {
-				blobpath, err := GetBlobsPath(strings.TrimPrefix(c.Args, "@"))
+				digest := strings.TrimPrefix(c.Args, "@")
+				if ib, ok := intermediateBlobs[digest]; ok {
+					p, err := GetBlobsPath(ib)
+					if err != nil {
+						return err
+					}
+
+					if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) {
+						// pass
+					} else if err != nil {
+						return err
+					} else {
+						fn(api.ProgressResponse{Status: fmt.Sprintf("using cached layer %s", ib)})
+						digest = ib
+					}
+				}
+
+				blobpath, err := GetBlobsPath(digest)
 				if err != nil {
 					return err
 				}
@@ -350,14 +367,14 @@ func CreateModel(ctx context.Context, name, modelFileDir, quantization string, m
 				}
 				defer blob.Close()
 
-				baseLayers, err = parseFromFile(ctx, blob, fn)
+				baseLayers, err = parseFromFile(ctx, blob, digest, fn)
 				if err != nil {
 					return err
 				}
 			} else if file, err := os.Open(realpath(modelFileDir, c.Args)); err == nil {
 				defer file.Close()
 
-				baseLayers, err = parseFromFile(ctx, file, fn)
+				baseLayers, err = parseFromFile(ctx, file, "", fn)
 				if err != nil {
 					return err
 				}
@@ -397,10 +414,17 @@ func CreateModel(ctx context.Context, name, modelFileDir, quantization string, m
 							return err
 						}
 
-						baseLayer.Layer, err = NewLayer(temp, baseLayer.Layer.MediaType)
+						layers, err := parseFromFile(ctx, temp, "", fn)
 						if err != nil {
 							return err
 						}
+
+						if len(layers) != 1 {
+							return errors.New("quantization failed")
+						}
+
+						baseLayer.Layer = layers[0].Layer
+						baseLayer.GGML = layers[0].GGML
 					}
 				}
 
@@ -415,19 +439,27 @@ func CreateModel(ctx context.Context, name, modelFileDir, quantization string, m
 				layers = append(layers, baseLayer.Layer)
 			}
 		case "license", "template", "system":
+			if c.Name != "license" {
+				// replace
+				layers = slices.DeleteFunc(layers, func(layer *Layer) bool {
+					if layer.MediaType != mediatype {
+						return false
+					}
+
+					if err := layer.Remove(); err != nil {
+						return false
+					}
+
+					return true
+				})
+			}
+
 			blob := strings.NewReader(c.Args)
 			layer, err := NewLayer(blob, mediatype)
 			if err != nil {
 				return err
 			}
 
-			if c.Name != "license" {
-				// replace
-				layers = slices.DeleteFunc(layers, func(layer *Layer) bool {
-					return layer.MediaType == mediatype
-				})
-			}
-
 			layers = append(layers, layer)
 		case "message":
 			role, content, ok := strings.Cut(c.Args, ": ")
@@ -546,26 +578,15 @@ func CreateModel(ctx context.Context, name, modelFileDir, quantization string, m
 		}
 	}
 
-	unref := make(map[string]struct{})
-	if manifest, _, err := GetManifest(ParseModelPath(name)); err == nil {
-		for _, layer := range manifest.Layers {
-			if !slices.Contains(digests, layer.Digest) {
-				unref[layer.Digest] = struct{}{}
-			}
-		}
-
-		if manifest.Config.Digest != layer.Digest {
-			unref[manifest.Config.Digest] = struct{}{}
-		}
-	}
+	old, _ := ParseNamedManifest(name)
 
 	fn(api.ProgressResponse{Status: "writing manifest"})
 	if err := WriteManifest(name, layer, layers); err != nil {
 		return err
 	}
 
-	if !envconfig.NoPrune {
-		if err := deleteUnusedLayers(nil, unref); err != nil {
+	if !envconfig.NoPrune && old != nil {
+		if err := old.RemoveLayers(); err != nil {
 			return err
 		}
 	}
@@ -637,7 +658,7 @@ func deleteUnusedLayers(skipModelPath *ModelPath, deleteMap map[string]struct{})
 		// save (i.e. delete from the deleteMap) any files used in other manifests
 		manifest, _, err := GetManifest(fmp)
 		if err != nil {
-			// nolint: nilerr
+			//nolint:nilerr
 			return nil
 		}
 
@@ -746,37 +767,6 @@ func PruneDirectory(path string) error {
 	return nil
 }
 
-func DeleteModel(name string) error {
-	mp := ParseModelPath(name)
-	manifest, _, err := GetManifest(mp)
-	if err != nil {
-		return err
-	}
-
-	deleteMap := make(map[string]struct{})
-	for _, layer := range manifest.Layers {
-		deleteMap[layer.Digest] = struct{}{}
-	}
-	deleteMap[manifest.Config.Digest] = struct{}{}
-
-	err = deleteUnusedLayers(&mp, deleteMap)
-	if err != nil {
-		return err
-	}
-
-	fp, err := mp.GetManifestPath()
-	if err != nil {
-		return err
-	}
-	err = os.Remove(fp)
-	if err != nil {
-		slog.Info(fmt.Sprintf("couldn't remove manifest file '%s': %v", fp, err))
-		return err
-	}
-
-	return nil
-}
-
 func PushModel(ctx context.Context, name string, regOpts *registryOptions, fn func(api.ProgressResponse)) error {
 	mp := ParseModelPath(name)
 	fn(api.ProgressResponse{Status: "retrieving manifest"})
@@ -863,23 +853,27 @@ func PullModel(ctx context.Context, name string, regOpts *registryOptions, fn fu
 	layers = append(layers, manifest.Layers...)
 	layers = append(layers, manifest.Config)
 
+	skipVerify := make(map[string]bool)
 	for _, layer := range layers {
-		if err := downloadBlob(
-			ctx,
-			downloadOpts{
-				mp:      mp,
-				digest:  layer.Digest,
-				regOpts: regOpts,
-				fn:      fn,
-			}); err != nil {
+		cacheHit, err := downloadBlob(ctx, downloadOpts{
+			mp:      mp,
+			digest:  layer.Digest,
+			regOpts: regOpts,
+			fn:      fn,
+		})
+		if err != nil {
 			return err
 		}
+		skipVerify[layer.Digest] = cacheHit
 		delete(deleteMap, layer.Digest)
 	}
 	delete(deleteMap, manifest.Config.Digest)
 
 	fn(api.ProgressResponse{Status: "verifying sha256 digest"})
 	for _, layer := range layers {
+		if skipVerify[layer.Digest] {
+			continue
+		}
 		if err := verifyBlob(layer.Digest); err != nil {
 			if errors.Is(err, errDigestMismatch) {
 				// something went wrong, delete the blob
@@ -966,7 +960,6 @@ var errUnauthorized = fmt.Errorf("unauthorized: access denied")
 func getTokenSubject(token string) string {
 	parts := strings.Split(token, ".")
 	if len(parts) != 3 {
-		slog.Error("jwt token does not contain 3 parts")
 		return ""
 	}
 
@@ -994,7 +987,7 @@ func getTokenSubject(token string) string {
 
 func makeRequestWithRetry(ctx context.Context, method string, requestURL *url.URL, headers http.Header, body io.ReadSeeker, regOpts *registryOptions) (*http.Response, error) {
 	anonymous := true // access will default to anonymous if no user is found associated with the public key
-	for i := 0; i < 2; i++ {
+	for range 2 {
 		resp, err := makeRequest(ctx, method, requestURL, headers, body, regOpts)
 		if err != nil {
 			if !errors.Is(err, context.Canceled) {

server/layer.go

@@ -80,7 +80,7 @@ func NewLayerFromLayer(digest, mediatype, from string) (*Layer, error) {
 	}, nil
 }
 
-func (l *Layer) Open() (io.ReadCloser, error) {
+func (l *Layer) Open() (io.ReadSeekCloser, error) {
 	blob, err := GetBlobsPath(l.Digest)
 	if err != nil {
 		return nil, err
@@ -88,3 +88,26 @@ func (l *Layer) Open() (io.ReadCloser, error) {
 
 	return os.Open(blob)
 }
+
+func (l *Layer) Remove() error {
+	ms, err := Manifests()
+	if err != nil {
+		return err
+	}
+
+	for _, m := range ms {
+		for _, layer := range append(m.Layers, m.Config) {
+			if layer.Digest == l.Digest {
+				// something is using this layer
+				return nil
+			}
+		}
+	}
+
+	blob, err := GetBlobsPath(l.Digest)
+	if err != nil {
+		return err
+	}
+
+	return os.Remove(blob)
+}

server/manifest.go

@@ -1,11 +1,12 @@
 package server
 
 import (
-	"bytes"
 	"crypto/sha256"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
+	"log/slog"
 	"os"
 	"path/filepath"
 
@@ -14,7 +15,10 @@ import (
 
 type Manifest struct {
 	ManifestV2
-	Digest string `json:"-"`
+
+	filepath string
+	fi       os.FileInfo
+	digest   string
 }
 
 func (m *Manifest) Size() (size int64) {
@@ -25,9 +29,34 @@ func (m *Manifest) Size() (size int64) {
 	return
 }
 
-func ParseNamedManifest(name model.Name) (*Manifest, error) {
-	if !name.IsFullyQualified() {
-		return nil, model.Unqualified(name)
+func (m *Manifest) Remove() error {
+	if err := os.Remove(m.filepath); err != nil {
+		return err
+	}
+
+	manifests, err := GetManifestPath()
+	if err != nil {
+		return err
+	}
+
+	return PruneDirectory(manifests)
+}
+
+func (m *Manifest) RemoveLayers() error {
+	for _, layer := range append(m.Layers, m.Config) {
+		if err := layer.Remove(); errors.Is(err, os.ErrNotExist) {
+			slog.Debug("layer does not exist", "digest", layer.Digest)
+		} else if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func ParseNamedManifest(n model.Name) (*Manifest, error) {
+	if !n.IsFullyQualified() {
+		return nil, model.Unqualified(n)
 	}
 
 	manifests, err := GetManifestPath()
@@ -35,45 +64,101 @@ func ParseNamedManifest(name model.Name) (*Manifest, error) {
 		return nil, err
 	}
 
-	var manifest ManifestV2
-	manifestfile, err := os.Open(filepath.Join(manifests, name.Filepath()))
+	p := filepath.Join(manifests, n.Filepath())
+
+	var m ManifestV2
+	f, err := os.Open(p)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	fi, err := f.Stat()
 	if err != nil {
 		return nil, err
 	}
 
 	sha256sum := sha256.New()
-	if err := json.NewDecoder(io.TeeReader(manifestfile, sha256sum)).Decode(&manifest); err != nil {
+	if err := json.NewDecoder(io.TeeReader(f, sha256sum)).Decode(&m); err != nil {
 		return nil, err
 	}
 
 	return &Manifest{
-		ManifestV2: manifest,
-		Digest:     fmt.Sprintf("%x", sha256sum.Sum(nil)),
+		ManifestV2: m,
+		filepath:   p,
+		fi:         fi,
+		digest:     fmt.Sprintf("%x", sha256sum.Sum(nil)),
 	}, nil
 }
 
-func WriteManifest(name string, config *Layer, layers []*Layer) error {
-	manifest := ManifestV2{
+func WriteManifest(name model.Name, config *Layer, layers []*Layer) error {
+	manifests, err := GetManifestPath()
+	if err != nil {
+		return err
+	}
+
+	p := filepath.Join(manifests, name.Filepath())
+	if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+		return err
+	}
+
+	f, err := os.Create(p)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	m := ManifestV2{
 		SchemaVersion: 2,
 		MediaType:     "application/vnd.docker.distribution.manifest.v2+json",
 		Config:        config,
 		Layers:        layers,
 	}
 
-	var b bytes.Buffer
-	if err := json.NewEncoder(&b).Encode(manifest); err != nil {
-		return err
-	}
-
-	modelpath := ParseModelPath(name)
-	manifestPath, err := modelpath.GetManifestPath()
-	if err != nil {
-		return err
-	}
-
-	if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil {
-		return err
-	}
-
-	return os.WriteFile(manifestPath, b.Bytes(), 0o644)
+	return json.NewEncoder(f).Encode(m)
+}
+
+func Manifests() (map[model.Name]*Manifest, error) {
+	manifests, err := GetManifestPath()
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO(mxyng): use something less brittle
+	matches, err := filepath.Glob(filepath.Join(manifests, "*", "*", "*", "*"))
+	if err != nil {
+		return nil, err
+	}
+
+	ms := make(map[model.Name]*Manifest)
+	for _, match := range matches {
+		fi, err := os.Stat(match)
+		if err != nil {
+			return nil, err
+		}
+
+		if !fi.IsDir() {
+			rel, err := filepath.Rel(manifests, match)
+			if err != nil {
+				slog.Warn("bad filepath", "path", match, "error", err)
+				continue
+			}
+
+			n := model.ParseNameFromFilepath(rel)
+			if !n.IsValid() {
+				slog.Warn("bad manifest name", "path", rel, "error", err)
+				continue
+			}
+
+			m, err := ParseNamedManifest(n)
+			if err != nil {
+				slog.Warn("bad manifest", "name", n, "error", err)
+				continue
+			}
+
+			ms[n] = m
+		}
+	}
+
+	return ms, nil
 }

server/manifest_test.go

@@ -0,0 +1,152 @@
+package server
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"slices"
+	"testing"
+
+	"github.com/ollama/ollama/envconfig"
+	"github.com/ollama/ollama/types/model"
+)
+
+func createManifest(t *testing.T, path, name string) {
+	t.Helper()
+
+	p := filepath.Join(path, "manifests", name)
+	if err := os.MkdirAll(filepath.Dir(p), 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	f, err := os.Create(p)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+
+	if err := json.NewEncoder(f).Encode(ManifestV2{}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestManifests(t *testing.T) {
+	cases := map[string]struct {
+		ps               []string
+		wantValidCount   int
+		wantInvalidCount int
+	}{
+		"empty": {},
+		"single": {
+			ps: []string{
+				filepath.Join("host", "namespace", "model", "tag"),
+			},
+			wantValidCount: 1,
+		},
+		"multiple": {
+			ps: []string{
+				filepath.Join("registry.ollama.ai", "library", "llama3", "latest"),
+				filepath.Join("registry.ollama.ai", "library", "llama3", "q4_0"),
+				filepath.Join("registry.ollama.ai", "library", "llama3", "q4_1"),
+				filepath.Join("registry.ollama.ai", "library", "llama3", "q8_0"),
+				filepath.Join("registry.ollama.ai", "library", "llama3", "q5_0"),
+				filepath.Join("registry.ollama.ai", "library", "llama3", "q5_1"),
+				filepath.Join("registry.ollama.ai", "library", "llama3", "q2_K"),
+				filepath.Join("registry.ollama.ai", "library", "llama3", "q3_K_S"),
+				filepath.Join("registry.ollama.ai", "library", "llama3", "q3_K_M"),
+				filepath.Join("registry.ollama.ai", "library", "llama3", "q3_K_L"),
+				filepath.Join("registry.ollama.ai", "library", "llama3", "q4_K_S"),
+				filepath.Join("registry.ollama.ai", "library", "llama3", "q4_K_M"),
+				filepath.Join("registry.ollama.ai", "library", "llama3", "q5_K_S"),
+				filepath.Join("registry.ollama.ai", "library", "llama3", "q5_K_M"),
+				filepath.Join("registry.ollama.ai", "library", "llama3", "q6_K"),
+			},
+			wantValidCount: 15,
+		},
+		"hidden": {
+			ps: []string{
+				filepath.Join("host", "namespace", "model", "tag"),
+				filepath.Join("host", "namespace", "model", ".hidden"),
+			},
+			wantValidCount:   1,
+			wantInvalidCount: 1,
+		},
+		"subdir": {
+			ps: []string{
+				filepath.Join("host", "namespace", "model", "tag", "one"),
+				filepath.Join("host", "namespace", "model", "tag", "another", "one"),
+			},
+			wantInvalidCount: 2,
+		},
+		"upper tag": {
+			ps: []string{
+				filepath.Join("host", "namespace", "model", "TAG"),
+			},
+			wantValidCount: 1,
+		},
+		"upper model": {
+			ps: []string{
+				filepath.Join("host", "namespace", "MODEL", "tag"),
+			},
+			wantValidCount: 1,
+		},
+		"upper namespace": {
+			ps: []string{
+				filepath.Join("host", "NAMESPACE", "model", "tag"),
+			},
+			wantValidCount: 1,
+		},
+		"upper host": {
+			ps: []string{
+				filepath.Join("HOST", "namespace", "model", "tag"),
+			},
+			wantValidCount: 1,
+		},
+	}
+
+	for n, wants := range cases {
+		t.Run(n, func(t *testing.T) {
+			d := t.TempDir()
+			t.Setenv("OLLAMA_MODELS", d)
+			envconfig.LoadConfig()
+
+			for _, p := range wants.ps {
+				createManifest(t, d, p)
+			}
+
+			ms, err := Manifests()
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			var ns []model.Name
+			for k := range ms {
+				ns = append(ns, k)
+			}
+
+			var gotValidCount, gotInvalidCount int
+			for _, p := range wants.ps {
+				n := model.ParseNameFromFilepath(p)
+				if n.IsValid() {
+					gotValidCount++
+				} else {
+					gotInvalidCount++
+				}
+
+				if !n.IsValid() && slices.Contains(ns, n) {
+					t.Errorf("unexpected invalid name: %s", p)
+				} else if n.IsValid() && !slices.Contains(ns, n) {
+					t.Errorf("missing valid name: %s", p)
+				}
+			}
+
+			if gotValidCount != wants.wantValidCount {
+				t.Errorf("got valid count %d, want %d", gotValidCount, wants.wantValidCount)
+			}
+
+			if gotInvalidCount != wants.wantInvalidCount {
+				t.Errorf("got invalid count %d, want %d", gotInvalidCount, wants.wantInvalidCount)
+			}
+		})
+	}
+}

server/model.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"log/slog"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -14,25 +15,26 @@ import (
 	"github.com/ollama/ollama/api"
 	"github.com/ollama/ollama/convert"
 	"github.com/ollama/ollama/llm"
+	"github.com/ollama/ollama/templates"
 	"github.com/ollama/ollama/types/model"
 )
 
-type layerWithGGML struct {
+var intermediateBlobs map[string]string = make(map[string]string)
+
+type layerGGML struct {
 	*Layer
 	*llm.GGML
 }
 
-func parseFromModel(ctx context.Context, name model.Name, fn func(api.ProgressResponse)) (layers []*layerWithGGML, err error) {
-	modelpath := ParseModelPath(name.String())
-	manifest, _, err := GetManifest(modelpath)
+func parseFromModel(ctx context.Context, name model.Name, fn func(api.ProgressResponse)) (layers []*layerGGML, err error) {
+	m, err := ParseNamedManifest(name)
 	switch {
 	case errors.Is(err, os.ErrNotExist):
 		if err := PullModel(ctx, name.String(), &registryOptions{}, fn); err != nil {
 			return nil, err
 		}
 
-		modelpath = ParseModelPath(name.String())
-		manifest, _, err = GetManifest(modelpath)
+		m, err = ParseNamedManifest(name)
 		if err != nil {
 			return nil, err
 		}
@@ -40,8 +42,8 @@ func parseFromModel(ctx context.Context, name model.Name, fn func(api.ProgressRe
 		return nil, err
 	}
 
-	for _, layer := range manifest.Layers {
-		layer, err := NewLayerFromLayer(layer.Digest, layer.MediaType, modelpath.GetShortTagname())
+	for _, layer := range m.Layers {
+		layer, err := NewLayerFromLayer(layer.Digest, layer.MediaType, name.DisplayShortest())
 		if err != nil {
 			return nil, err
 		}
@@ -66,17 +68,16 @@ func parseFromModel(ctx context.Context, name model.Name, fn func(api.ProgressRe
 				return nil, err
 			}
 
-			layers = append(layers, &layerWithGGML{layer, ggml})
+			layers = append(layers, &layerGGML{layer, ggml})
 		default:
-			layers = append(layers, &layerWithGGML{layer, nil})
+			layers = append(layers, &layerGGML{layer, nil})
 		}
-
 	}
 
 	return layers, nil
 }
 
-func parseFromZipFile(_ context.Context, file *os.File, fn func(api.ProgressResponse)) (layers []*layerWithGGML, err error) {
+func parseFromZipFile(_ context.Context, file *os.File, digest string, fn func(api.ProgressResponse)) (layers []*layerGGML, err error) {
 	stat, err := file.Stat()
 	if err != nil {
 		return nil, err
@@ -165,16 +166,11 @@ func parseFromZipFile(_ context.Context, file *os.File, fn func(api.ProgressResp
 	}
 
 	layer, err := NewLayer(temp, "application/vnd.ollama.image.model")
-	if err != nil {
-		return nil, fmt.Errorf("aaa: %w", err)
-	}
-
-	blobpath, err := GetBlobsPath(layer.Digest)
 	if err != nil {
 		return nil, err
 	}
 
-	bin, err := os.Open(blobpath)
+	bin, err := layer.Open()
 	if err != nil {
 		return nil, err
 	}
@@ -185,16 +181,13 @@ func parseFromZipFile(_ context.Context, file *os.File, fn func(api.ProgressResp
 		return nil, err
 	}
 
-	layer, err = NewLayerFromLayer(layer.Digest, layer.MediaType, "")
-	if err != nil {
-		return nil, err
-	}
+	layers = append(layers, &layerGGML{layer, ggml})
 
-	layers = append(layers, &layerWithGGML{layer, ggml})
-	return layers, nil
+	intermediateBlobs[digest] = layer.Digest
+	return detectChatTemplate(layers)
 }
 
-func parseFromFile(ctx context.Context, file *os.File, fn func(api.ProgressResponse)) (layers []*layerWithGGML, err error) {
+func parseFromFile(ctx context.Context, file *os.File, digest string, fn func(api.ProgressResponse)) (layers []*layerGGML, err error) {
 	sr := io.NewSectionReader(file, 0, 512)
 	contentType, err := detectContentType(sr)
 	if err != nil {
@@ -205,7 +198,7 @@ func parseFromFile(ctx context.Context, file *os.File, fn func(api.ProgressRespo
 	case "gguf", "ggla":
 		// noop
 	case "application/zip":
-		return parseFromZipFile(ctx, file, fn)
+		return parseFromZipFile(ctx, file, digest, fn)
 	default:
 		return nil, fmt.Errorf("unsupported content type: %s", contentType)
 	}
@@ -236,10 +229,30 @@ func parseFromFile(ctx context.Context, file *os.File, fn func(api.ProgressRespo
 			return nil, err
 		}
 
-		layers = append(layers, &layerWithGGML{layer, ggml})
+		layers = append(layers, &layerGGML{layer, ggml})
 		offset = n
 	}
 
+	return detectChatTemplate(layers)
+}
+
+func detectChatTemplate(layers []*layerGGML) ([]*layerGGML, error) {
+	for _, layer := range layers {
+		if s := layer.GGML.KV().ChatTemplate(); s != "" {
+			if t, err := templates.NamedTemplate(s); err != nil {
+				slog.Debug("template detection", "error", err)
+			} else {
+				tmpl, err := NewLayer(t.Reader(), "application/vnd.ollama.image.template")
+				if err != nil {
+					return nil, err
+				}
+
+				tmpl.status = fmt.Sprintf("using autodetected template %s", t.Name)
+				layers = append(layers, &layerGGML{tmpl, nil})
+			}
+		}
+	}
+
 	return layers, nil
 }
 

server/modelpath.go

@@ -8,6 +8,8 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+
+	"github.com/ollama/ollama/envconfig"
 )
 
 type ModelPath struct {
@@ -104,14 +106,7 @@ func (mp ModelPath) GetShortTagname() string {
 // modelsDir returns the value of the OLLAMA_MODELS environment variable or the user's home directory if OLLAMA_MODELS is not set.
 // The models directory is where Ollama stores its model files and manifests.
 func modelsDir() (string, error) {
-	if models, exists := os.LookupEnv("OLLAMA_MODELS"); exists {
-		return models, nil
-	}
-	home, err := os.UserHomeDir()
-	if err != nil {
-		return "", err
-	}
-	return filepath.Join(home, ".ollama", "models"), nil
+	return envconfig.ModelsDir, nil
 }
 
 // GetManifestPath returns the path to the manifest file for the given model path, it is up to the caller to create the directory if it does not exist.

server/modelpath_test.go

@@ -6,12 +6,15 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/ollama/ollama/envconfig"
 )
 
 func TestGetBlobsPath(t *testing.T) {
 	// GetBlobsPath expects an actual directory to exist
 	dir, err := os.MkdirTemp("", "ollama-test")
-	assert.Nil(t, err)
+	require.NoError(t, err)
 	defer os.RemoveAll(dir)
 
 	tests := []struct {
@@ -60,10 +63,11 @@ func TestGetBlobsPath(t *testing.T) {
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Setenv("OLLAMA_MODELS", dir)
+			envconfig.LoadConfig()
 
 			got, err := GetBlobsPath(tc.digest)
 
-			assert.ErrorIs(t, tc.err, err, tc.name)
+			require.ErrorIs(t, tc.err, err, tc.name)
 			assert.Equal(t, tc.expected, got, tc.name)
 		})
 	}