JSON mode: add `"format" as an api parameter (#1051)

* add `"format": "json"` as an API parameter
---------
Co-authored-by: Bruce MacDonald <brucewmacdonald@gmail.com>

.dockerignore

@@ -1,7 +1,8 @@
-build
-llama/build
-.venv
 .vscode
 ollama
 app
-web
+dist
+scripts
+llm/llama.cpp/ggml
+llm/llama.cpp/gguf
+.env

.gitignore

@@ -5,3 +5,4 @@
 .swp
 dist
 ollama
+ggml-metal.metal

.gitmodules

@@ -0,0 +1,10 @@
+[submodule "llm/llama.cpp/ggml"]
+    path = llm/llama.cpp/ggml
+    url = https://github.com/ggerganov/llama.cpp.git
+    ignore = dirty
+    shallow = true
+[submodule "llm/llama.cpp/gguf"]
+    path = llm/llama.cpp/gguf
+    url = https://github.com/ggerganov/llama.cpp.git
+    ignore = dirty
+    shallow = true

Dockerfile

@@ -1,15 +1,23 @@
-FROM golang:1.20
-WORKDIR /go/src/github.com/jmorganca/ollama
-COPY . .
-RUN CGO_ENABLED=1 go build -ldflags '-linkmode external -extldflags "-static"' .
+FROM nvidia/cuda:11.8.0-devel-ubuntu22.04
 
-FROM alpine
+ARG TARGETARCH
+ARG GOFLAGS="'-ldflags=-w -s'"
+
+WORKDIR /go/src/github.com/jmorganca/ollama
+RUN apt-get update && apt-get install -y git build-essential cmake
+ADD https://dl.google.com/go/go1.21.3.linux-$TARGETARCH.tar.gz /tmp/go1.21.3.tar.gz
+RUN mkdir -p /usr/local && tar xz -C /usr/local </tmp/go1.21.3.tar.gz
+
+COPY . .
+ENV GOARCH=$TARGETARCH
+ENV GOFLAGS=$GOFLAGS
+RUN /usr/local/go/bin/go generate ./... \
+    && /usr/local/go/bin/go build .
+
+FROM ubuntu:22.04
+RUN apt-get update && apt-get install -y ca-certificates
 COPY --from=0 /go/src/github.com/jmorganca/ollama/ollama /bin/ollama
 EXPOSE 11434
-ARG USER=ollama
-ARG GROUP=ollama
-RUN addgroup -g 1000 $GROUP && adduser -u 1000 -DG $GROUP $USER
-USER $USER:$GROUP
-ENTRYPOINT ["/bin/ollama"]
 ENV OLLAMA_HOST 0.0.0.0
+ENTRYPOINT ["/bin/ollama"]
 CMD ["serve"]

Dockerfile.build

@@ -0,0 +1,31 @@
+# centos7 amd64 dependencies
+FROM --platform=linux/amd64 nvidia/cuda:11.3.1-devel-centos7 AS base-amd64
+RUN yum install -y https://repo.ius.io/ius-release-el7.rpm centos-release-scl && \
+    yum update -y && \
+    yum install -y devtoolset-10-gcc devtoolset-10-gcc-c++ git236 wget
+RUN wget "https://github.com/Kitware/CMake/releases/download/v3.27.6/cmake-3.27.6-linux-x86_64.sh" -O cmake-installer.sh && chmod +x cmake-installer.sh && ./cmake-installer.sh --skip-license --prefix=/usr/local
+ENV PATH /opt/rh/devtoolset-10/root/usr/bin:$PATH
+
+# centos8 arm64 dependencies
+FROM --platform=linux/arm64 nvidia/cuda-arm64:11.3.1-devel-centos8 AS base-arm64
+RUN sed -i -e 's/mirrorlist/#mirrorlist/g' -e 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
+RUN yum install -y git cmake
+
+FROM base-${TARGETARCH}
+ARG TARGETARCH
+ARG GOFLAGS="'-ldflags -w -s'"
+
+# install go
+ADD https://dl.google.com/go/go1.21.3.linux-$TARGETARCH.tar.gz /tmp/go1.21.3.tar.gz
+RUN mkdir -p /usr/local && tar xz -C /usr/local </tmp/go1.21.3.tar.gz
+
+# build the final binary
+WORKDIR /go/src/github.com/jmorganca/ollama
+COPY . .
+
+ENV GOOS=linux
+ENV GOARCH=$TARGETARCH
+ENV GOFLAGS=$GOFLAGS
+
+RUN /usr/local/go/bin/go generate ./... && \
+    /usr/local/go/bin/go build .

README.md

@@ -9,19 +9,31 @@
 
 [![Discord](https://dcbadge.vercel.app/api/server/ollama?style=flat&compact=true)](https://discord.gg/ollama)
 
-> Note: Ollama is in early preview. Please report any issues you find.
+Get up and running with large language models locally.
 
-Run, create, and share large language models (LLMs).
+### macOS
 
-## Download
+[Download](https://ollama.ai/download/Ollama-darwin.zip)
 
-- [Download](https://ollama.ai/download) for macOS on Apple Silicon (Intel coming soon)
-- Download for Windows and Linux (coming soon)
-- Build [from source](#building)
+### Windows
+
+Coming soon!
+
+### Linux & WSL2
+
+```
+curl https://ollama.ai/install.sh | sh
+```
+
+[Manual install instructions](https://github.com/jmorganca/ollama/blob/main/docs/linux.md)
+
+### Docker
+
+The official [Ollama Docker image](https://hub.docker.com/r/ollama/ollama) `ollama/ollama` is available on Docker Hub.
 
 ## Quickstart
 
-To run and chat with [Llama 2](https://ai.meta.com/llama), the new model by Meta:
+To run and chat with [Llama 2](https://ollama.ai/library/llama2):
 
 ```
 ollama run llama2
@@ -29,32 +41,54 @@ ollama run llama2
 
 ## Model library
 
-`ollama` includes a library of open-source models:
+Ollama supports a list of open-source models available on [ollama.ai/library](https://ollama.ai/library 'ollama model library')
 
-| Model                    | Parameters | Size  | Download                    |
-| ------------------------ | ---------- | ----- | --------------------------- |
-| Llama2                   | 7B         | 3.8GB | `ollama pull llama2`        |
-| Llama2 13B               | 13B        | 7.3GB | `ollama pull llama2:13b`    |
-| Orca Mini                | 3B         | 1.9GB | `ollama pull orca`          |
-| Vicuna                   | 7B         | 3.8GB | `ollama pull vicuna`        |
-| Nous-Hermes              | 13B        | 7.3GB | `ollama pull nous-hermes`   |
-| Wizard Vicuna Uncensored | 13B        | 7.3GB | `ollama pull wizard-vicuna` |
+Here are some example open-source models that can be downloaded:
+
+| Model              | Parameters | Size  | Download                       |
+| ------------------ | ---------- | ----- | ------------------------------ |
+| Mistral            | 7B         | 4.1GB | `ollama run mistral`           |
+| Llama 2            | 7B         | 3.8GB | `ollama run llama2`            |
+| Code Llama         | 7B         | 3.8GB | `ollama run codellama`         |
+| Llama 2 Uncensored | 7B         | 3.8GB | `ollama run llama2-uncensored` |
+| Llama 2 13B        | 13B        | 7.3GB | `ollama run llama2:13b`        |
+| Llama 2 70B        | 70B        | 39GB  | `ollama run llama2:70b`        |
+| Orca Mini          | 3B         | 1.9GB | `ollama run orca-mini`         |
+| Vicuna             | 7B         | 3.8GB | `ollama run vicuna`            |
 
 > Note: You should have at least 8 GB of RAM to run the 3B models, 16 GB to run the 7B models, and 32 GB to run the 13B models.
 
-## Examples
+## Customize your own model
 
-### Run a model
+### Import from GGUF
 
-```
-ollama run llama2
->>> hi
-Hello! How can I help you today?
-```
+Ollama supports importing GGUF models in the Modelfile:
 
-### Create a custom model
+1. Create a file named `Modelfile`, with a `FROM` instruction with the local filepath to the model you want to import.
 
-Pull a base model:
+   ```
+   FROM ./vicuna-33b.Q4_0.gguf
+   ```
+
+2. Create the model in Ollama
+
+   ```
+   ollama create example -f Modelfile
+   ```
+
+3. Run the model
+
+   ```
+   ollama run example
+   ```
+
+### Import from PyTorch or Safetensors
+
+See the [guide](docs/import.md) on importing models for more information.
+
+### Customize a prompt
+
+Models from the Ollama library can be customized with a prompt. For example, to customize the `llama2` model:
 
 ```
 ollama pull llama2
@@ -83,44 +117,84 @@ ollama run mario
 Hello! It's your friend Mario.
 ```
 
-For more examples, see the [examples](./examples) directory.
+For more examples, see the [examples](examples) directory. For more information on working with a Modelfile, see the [Modelfile](docs/modelfile.md) documentation.
 
-### Pull a model from the registry
+## CLI Reference
+
+### Create a model
+
+`ollama create` is used to create a model from a Modelfile.
+
+### Pull a model
 
 ```
-ollama pull orca
+ollama pull llama2
 ```
 
-### Listing local models
+> This command can also be used to update a local model. Only the diff will be pulled.
+
+### Remove a model
+
+```
+ollama rm llama2
+```
+
+### Copy a model
+
+```
+ollama cp llama2 my-llama2
+```
+
+### Multiline input
+
+For multiline input, you can wrap text with `"""`:
+
+```
+>>> """Hello,
+... world!
+... """
+I'm a basic program that prints the famous "Hello, world!" message to the console.
+```
+
+### Pass in prompt as arguments
+
+```
+$ ollama run llama2 "Summarize this file: $(cat README.md)"
+ Ollama is a lightweight, extensible framework for building and running language models on the local machine. It provides a simple API for creating, running, and managing models, as well as a library of pre-built models that can be easily used in a variety of applications.
+```
+
+### List models on your computer
 
 ```
 ollama list
 ```
 
-## Model packages
+### Start Ollama
 
-### Overview
-
-Ollama bundles model weights, configuration, and data into a single package, defined by a [Modelfile](./docs/modelfile.md).
-
-<picture>
-  <source media="(prefers-color-scheme: dark)" height="480" srcset="https://github.com/jmorganca/ollama/assets/251292/2fd96b5f-191b-45c1-9668-941cfad4eb70">
-  <img alt="logo" height="480" src="https://github.com/jmorganca/ollama/assets/251292/2fd96b5f-191b-45c1-9668-941cfad4eb70">
-</picture>
+`ollama serve` is used when you want to start ollama without running the desktop application.
 
 ## Building
 
+Install `cmake` and `go`:
+
 ```
+brew install cmake go
+```
+
+Then generate dependencies and build:
+
+```
+go generate ./...
 go build .
 ```
 
-To run it start the server:
+Next, start the server:
 
 ```
-./ollama serve &
+./ollama serve
 ```
 
-Finally, run a model!
+Finally, in a separate shell, run a model:
 
 ```
 ./ollama run llama2
@@ -128,10 +202,54 @@ Finally, run a model!
 
 ## REST API
 
-### `POST /api/generate`
-
-Generate text from a model.
+Ollama has a REST API for running and managing models.
+For example, to generate text from a model:
 
 ```
-curl -X POST http://localhost:11434/api/generate -d '{"model": "llama2", "prompt":"Why is the sky blue?"}'
+curl -X POST http://localhost:11434/api/generate -d '{
+  "model": "llama2",
+  "prompt":"Why is the sky blue?"
+}'
 ```
+
+See the [API documentation](./docs/api.md) for all endpoints.
+
+## Community Integrations
+
+### Web & Desktop
+
+- [HTML UI](https://github.com/rtcfirefly/ollama-ui)
+- [Chatbot UI](https://github.com/ivanfioravanti/chatbot-ollama)
+- [Typescript UI](https://github.com/ollama-interface/Ollama-Gui?tab=readme-ov-file)
+- [Minimalistic React UI for Ollama Models](https://github.com/richawo/minimal-llm-ui)
+- [Web UI](https://github.com/ollama-webui/ollama-webui)
+- [Ollamac](https://github.com/kevinhermawan/Ollamac)
+
+### Terminal
+
+- [oterm](https://github.com/ggozad/oterm)
+- [Ellama Emacs client](https://github.com/s-kostyaev/ellama)
+- [Emacs client](https://github.com/zweifisch/ollama)
+- [gen.nvim](https://github.com/David-Kunz/gen.nvim)
+- [gptel Emacs client](https://github.com/karthink/gptel)
+
+### Libraries
+
+- [LangChain](https://python.langchain.com/docs/integrations/llms/ollama) and [LangChain.js](https://js.langchain.com/docs/modules/model_io/models/llms/integrations/ollama) with [example](https://js.langchain.com/docs/use_cases/question_answering/local_retrieval_qa)
+- [LlamaIndex](https://gpt-index.readthedocs.io/en/stable/examples/llm/ollama.html)
+- [LiteLLM](https://github.com/BerriAI/litellm)
+- [OllamaSharp for .NET](https://github.com/awaescher/OllamaSharp)
+- [Ollama-rs for Rust](https://github.com/pepperoni21/ollama-rs)
+- [Ollama4j for Java](https://github.com/amithkoujalgi/ollama4j)
+- [ModelFusion Typescript Library](https://modelfusion.dev/integration/model-provider/ollama)
+
+### Extensions & Plugins
+
+- [Raycast extension](https://github.com/MassimilianoPasquini97/raycast_ollama)
+- [Discollama](https://github.com/mxyng/discollama) (Discord bot inside the Ollama discord channel)
+- [Continue](https://github.com/continuedev/continue)
+- [Obsidian Ollama plugin](https://github.com/hinterdupfinger/obsidian-ollama)
+- [Logseq Ollama plugin](https://github.com/omagdy7/ollama-logseq)
+- [Dagger Chatbot](https://github.com/samalba/dagger-chatbot)
+- [Discord AI Bot](https://github.com/mekb-turtle/discord-ai-bot)
+- [Hass Ollama Conversation](https://github.com/ej52/hass-ollama-conversation)

api/client.go

@@ -7,18 +7,24 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
+	"os"
+	"runtime"
+	"strings"
+
+	"github.com/jmorganca/ollama/format"
+	"github.com/jmorganca/ollama/version"
 )
 
 type Client struct {
-	base    url.URL
-	HTTP    http.Client
-	Headers http.Header
+	base *url.URL
+	http http.Client
 }
 
 func checkError(resp *http.Response, body []byte) error {
-	if resp.StatusCode >= 200 && resp.StatusCode < 400 {
+	if resp.StatusCode < http.StatusBadRequest {
 		return nil
 	}
 
@@ -33,16 +39,56 @@ func checkError(resp *http.Response, body []byte) error {
 	return apiError
 }
 
-func NewClient(hosts ...string) *Client {
-	host := "127.0.0.1:11434"
-	if len(hosts) > 0 {
-		host = hosts[0]
+func ClientFromEnvironment() (*Client, error) {
+	defaultPort := "11434"
+
+	scheme, hostport, ok := strings.Cut(os.Getenv("OLLAMA_HOST"), "://")
+	switch {
+	case !ok:
+		scheme, hostport = "http", os.Getenv("OLLAMA_HOST")
+	case scheme == "http":
+		defaultPort = "80"
+	case scheme == "https":
+		defaultPort = "443"
 	}
 
-	return &Client{
-		base: url.URL{Scheme: "http", Host: host},
-		HTTP: http.Client{},
+	// trim trailing slashes
+	hostport = strings.TrimRight(hostport, "/")
+
+	host, port, err := net.SplitHostPort(hostport)
+	if err != nil {
+		host, port = "127.0.0.1", defaultPort
+		if ip := net.ParseIP(strings.Trim(hostport, "[]")); ip != nil {
+			host = ip.String()
+		} else if hostport != "" {
+			host = hostport
+		}
 	}
+
+	client := Client{
+		base: &url.URL{
+			Scheme: scheme,
+			Host:   net.JoinHostPort(host, port),
+		},
+	}
+
+	mockRequest, err := http.NewRequest(http.MethodHead, client.base.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	proxyURL, err := http.ProxyFromEnvironment(mockRequest)
+	if err != nil {
+		return nil, err
+	}
+
+	client.http = http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyURL(proxyURL),
+		},
+	}
+
+	return &client, nil
 }
 
 func (c *Client) do(ctx context.Context, method, path string, reqData, respData any) error {
@@ -57,21 +103,17 @@ func (c *Client) do(ctx context.Context, method, path string, reqData, respData
 		reqBody = bytes.NewReader(data)
 	}
 
-	url := c.base.JoinPath(path).String()
-
-	req, err := http.NewRequestWithContext(ctx, method, url, reqBody)
+	requestURL := c.base.JoinPath(path)
+	request, err := http.NewRequestWithContext(ctx, method, requestURL.String(), reqBody)
 	if err != nil {
 		return err
 	}
 
-	req.Header.Set("Content-Type", "application/json")
-	req.Header.Set("Accept", "application/json")
+	request.Header.Set("Content-Type", "application/json")
+	request.Header.Set("Accept", "application/json")
+	request.Header.Set("User-Agent", fmt.Sprintf("ollama/%s (%s %s) Go/%s", version.Version, runtime.GOARCH, runtime.GOOS, runtime.Version()))
 
-	for k, v := range c.Headers {
-		req.Header[k] = v
-	}
-
-	respObj, err := c.HTTP.Do(req)
+	respObj, err := c.http.Do(request)
 	if err != nil {
 		return err
 	}
@@ -94,6 +136,8 @@ func (c *Client) do(ctx context.Context, method, path string, reqData, respData
 	return nil
 }
 
+const maxBufferSize = 512 * format.KiloByte
+
 func (c *Client) stream(ctx context.Context, method, path string, data any, fn func([]byte) error) error {
 	var buf *bytes.Buffer
 	if data != nil {
@@ -105,21 +149,26 @@ func (c *Client) stream(ctx context.Context, method, path string, data any, fn f
 		buf = bytes.NewBuffer(bts)
 	}
 
-	request, err := http.NewRequestWithContext(ctx, method, c.base.JoinPath(path).String(), buf)
+	requestURL := c.base.JoinPath(path)
+	request, err := http.NewRequestWithContext(ctx, method, requestURL.String(), buf)
 	if err != nil {
 		return err
 	}
 
 	request.Header.Set("Content-Type", "application/json")
-	request.Header.Set("Accept", "application/json")
+	request.Header.Set("Accept", "application/x-ndjson")
+	request.Header.Set("User-Agent", fmt.Sprintf("ollama/%s (%s %s) Go/%s", version.Version, runtime.GOARCH, runtime.GOOS, runtime.Version()))
 
-	response, err := http.DefaultClient.Do(request)
+	response, err := c.http.Do(request)
 	if err != nil {
 		return err
 	}
 	defer response.Body.Close()
 
 	scanner := bufio.NewScanner(response.Body)
+	// increase the buffer size to avoid running out of space
+	scanBuf := make([]byte, 0, maxBufferSize)
+	scanner.Buffer(scanBuf, maxBufferSize)
 	for scanner.Scan() {
 		var errorResponse struct {
 			Error string `json:"error,omitempty"`
@@ -131,10 +180,10 @@ func (c *Client) stream(ctx context.Context, method, path string, data any, fn f
 		}
 
 		if errorResponse.Error != "" {
-			return fmt.Errorf("stream: %s", errorResponse.Error)
+			return fmt.Errorf(errorResponse.Error)
 		}
 
-		if response.StatusCode >= 400 {
+		if response.StatusCode >= http.StatusBadRequest {
 			return StatusError{
 				StatusCode:   response.StatusCode,
 				Status:       response.Status,
@@ -189,11 +238,11 @@ func (c *Client) Push(ctx context.Context, req *PushRequest, fn PushProgressFunc
 	})
 }
 
-type CreateProgressFunc func(CreateProgress) error
+type CreateProgressFunc func(ProgressResponse) error
 
 func (c *Client) Create(ctx context.Context, req *CreateRequest, fn CreateProgressFunc) error {
 	return c.stream(ctx, http.MethodPost, "/api/create", req, func(bts []byte) error {
-		var resp CreateProgress
+		var resp ProgressResponse
 		if err := json.Unmarshal(bts, &resp); err != nil {
 			return err
 		}
@@ -223,3 +272,18 @@ func (c *Client) Delete(ctx context.Context, req *DeleteRequest) error {
 	}
 	return nil
 }
+
+func (c *Client) Show(ctx context.Context, req *ShowRequest) (*ShowResponse, error) {
+	var resp ShowResponse
+	if err := c.do(ctx, http.MethodPost, "/api/show", req, &resp); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
+func (c *Client) Heartbeat(ctx context.Context) error {
+	if err := c.do(ctx, http.MethodHead, "/", nil, nil); err != nil {
+		return err
+	}
+	return nil
+}

api/client.py

@@ -0,0 +1,225 @@
+import os
+import json
+import requests
+
+BASE_URL = os.environ.get('OLLAMA_HOST', 'http://localhost:11434')
+
+# Generate a response for a given prompt with a provided model. This is a streaming endpoint, so will be a series of responses.
+# The final response object will include statistics and additional data from the request. Use the callback function to override
+# the default handler.
+def generate(model_name, prompt, system=None, template=None, context=None, options=None, callback=None):
+    try:
+        url = f"{BASE_URL}/api/generate"
+        payload = {
+            "model": model_name, 
+            "prompt": prompt, 
+            "system": system, 
+            "template": template, 
+            "context": context, 
+            "options": options
+        }
+        
+        # Remove keys with None values
+        payload = {k: v for k, v in payload.items() if v is not None}
+        
+        with requests.post(url, json=payload, stream=True) as response:
+            response.raise_for_status()
+            
+            # Creating a variable to hold the context history of the final chunk
+            final_context = None
+            
+            # Variable to hold concatenated response strings if no callback is provided
+            full_response = ""
+
+            # Iterating over the response line by line and displaying the details
+            for line in response.iter_lines():
+                if line:
+                    # Parsing each line (JSON chunk) and extracting the details
+                    chunk = json.loads(line)
+                    
+                    # If a callback function is provided, call it with the chunk
+                    if callback:
+                        callback(chunk)
+                    else:
+                        # If this is not the last chunk, add the "response" field value to full_response and print it
+                        if not chunk.get("done"):
+                            response_piece = chunk.get("response", "")
+                            full_response += response_piece
+                            print(response_piece, end="", flush=True)
+                    
+                    # Check if it's the last chunk (done is true)
+                    if chunk.get("done"):
+                        final_context = chunk.get("context")
+            
+            # Return the full response and the final context
+            return full_response, final_context
+    except requests.exceptions.RequestException as e:
+        print(f"An error occurred: {e}")
+        return None, None
+
+# Create a model from a Modelfile. Use the callback function to override the default handler.
+def create(model_name, model_path, callback=None):
+    try:
+        url = f"{BASE_URL}/api/create"
+        payload = {"name": model_name, "path": model_path}
+        
+        # Making a POST request with the stream parameter set to True to handle streaming responses
+        with requests.post(url, json=payload, stream=True) as response:
+            response.raise_for_status()
+
+            # Iterating over the response line by line and displaying the status
+            for line in response.iter_lines():
+                if line:
+                    # Parsing each line (JSON chunk) and extracting the status
+                    chunk = json.loads(line)
+
+                    if callback:
+                        callback(chunk)
+                    else:
+                        print(f"Status: {chunk.get('status')}")
+    except requests.exceptions.RequestException as e:
+        print(f"An error occurred: {e}")
+
+# Pull a model from a the model registry. Cancelled pulls are resumed from where they left off, and multiple
+# calls to will share the same download progress. Use the callback function to override the default handler.
+def pull(model_name, insecure=False, callback=None):
+    try:
+        url = f"{BASE_URL}/api/pull"
+        payload = {
+            "name": model_name,
+            "insecure": insecure
+        }
+
+        # Making a POST request with the stream parameter set to True to handle streaming responses
+        with requests.post(url, json=payload, stream=True) as response:
+            response.raise_for_status()
+
+            # Iterating over the response line by line and displaying the details
+            for line in response.iter_lines():
+                if line:
+                    # Parsing each line (JSON chunk) and extracting the details
+                    chunk = json.loads(line)
+
+                    # If a callback function is provided, call it with the chunk
+                    if callback:
+                        callback(chunk)
+                    else:
+                        # Print the status message directly to the console
+                        print(chunk.get('status', ''), end='', flush=True)
+                    
+                    # If there's layer data, you might also want to print that (adjust as necessary)
+                    if 'digest' in chunk:
+                        print(f" - Digest: {chunk['digest']}", end='', flush=True)
+                        print(f" - Total: {chunk['total']}", end='', flush=True)
+                        print(f" - Completed: {chunk['completed']}", end='\n', flush=True)
+                    else:
+                        print()
+    except requests.exceptions.RequestException as e:
+        print(f"An error occurred: {e}")
+
+# Push a model to the model registry. Use the callback function to override the default handler.
+def push(model_name, insecure=False, callback=None):
+    try:
+        url = f"{BASE_URL}/api/push"
+        payload = {
+            "name": model_name,
+            "insecure": insecure
+        }
+
+        # Making a POST request with the stream parameter set to True to handle streaming responses
+        with requests.post(url, json=payload, stream=True) as response:
+            response.raise_for_status()
+
+            # Iterating over the response line by line and displaying the details
+            for line in response.iter_lines():
+                if line:
+                    # Parsing each line (JSON chunk) and extracting the details
+                    chunk = json.loads(line)
+
+                    # If a callback function is provided, call it with the chunk
+                    if callback:
+                        callback(chunk)
+                    else:
+                        # Print the status message directly to the console
+                        print(chunk.get('status', ''), end='', flush=True)
+                    
+                    # If there's layer data, you might also want to print that (adjust as necessary)
+                    if 'digest' in chunk:
+                        print(f" - Digest: {chunk['digest']}", end='', flush=True)
+                        print(f" - Total: {chunk['total']}", end='', flush=True)
+                        print(f" - Completed: {chunk['completed']}", end='\n', flush=True)
+                    else:
+                        print()
+    except requests.exceptions.RequestException as e:
+        print(f"An error occurred: {e}")
+
+# List models that are available locally.
+def list():
+    try:
+        response = requests.get(f"{BASE_URL}/api/tags")
+        response.raise_for_status()
+        data = response.json()
+        models = data.get('models', [])
+        return models
+
+    except requests.exceptions.RequestException as e:
+        print(f"An error occurred: {e}")
+        return None
+
+# Copy a model. Creates a model with another name from an existing model.
+def copy(source, destination):
+    try:
+        # Create the JSON payload
+        payload = {
+            "source": source,
+            "destination": destination
+        }
+        
+        response = requests.post(f"{BASE_URL}/api/copy", json=payload)
+        response.raise_for_status()
+        
+        # If the request was successful, return a message indicating that the copy was successful
+        return "Copy successful"
+
+    except requests.exceptions.RequestException as e:
+        print(f"An error occurred: {e}")
+        return None
+
+# Delete a model and its data.
+def delete(model_name):
+    try:
+        url = f"{BASE_URL}/api/delete"
+        payload = {"name": model_name}
+        response = requests.delete(url, json=payload)
+        response.raise_for_status()
+        return "Delete successful"
+    except requests.exceptions.RequestException as e:
+        print(f"An error occurred: {e}")
+        return None
+
+# Show info about a model.
+def show(model_name):
+    try:
+        url = f"{BASE_URL}/api/show"
+        payload = {"name": model_name}
+        response = requests.post(url, json=payload)
+        response.raise_for_status()
+        
+        # Parse the JSON response and return it
+        data = response.json()
+        return data
+    except requests.exceptions.RequestException as e:
+        print(f"An error occurred: {e}")
+        return None
+
+def heartbeat():
+    try:
+        url = f"{BASE_URL}/"
+        response = requests.head(url)
+        response.raise_for_status()
+        return "Ollama is running"
+    except requests.exceptions.RequestException as e:
+        print(f"An error occurred: {e}")
+        return "Ollama is not running"
+
+

api/client_test.go

@@ -0,0 +1,43 @@
+package api
+
+import "testing"
+
+func TestClientFromEnvironment(t *testing.T) {
+	type testCase struct {
+		value  string
+		expect string
+		err    error
+	}
+
+	testCases := map[string]*testCase{
+		"empty":                      {value: "", expect: "http://127.0.0.1:11434"},
+		"only address":               {value: "1.2.3.4", expect: "http://1.2.3.4:11434"},
+		"only port":                  {value: ":1234", expect: "http://:1234"},
+		"address and port":           {value: "1.2.3.4:1234", expect: "http://1.2.3.4:1234"},
+		"scheme http and address":    {value: "http://1.2.3.4", expect: "http://1.2.3.4:80"},
+		"scheme https and address":   {value: "https://1.2.3.4", expect: "https://1.2.3.4:443"},
+		"scheme, address, and port":  {value: "https://1.2.3.4:1234", expect: "https://1.2.3.4:1234"},
+		"hostname":                   {value: "example.com", expect: "http://example.com:11434"},
+		"hostname and port":          {value: "example.com:1234", expect: "http://example.com:1234"},
+		"scheme http and hostname":   {value: "http://example.com", expect: "http://example.com:80"},
+		"scheme https and hostname":  {value: "https://example.com", expect: "https://example.com:443"},
+		"scheme, hostname, and port": {value: "https://example.com:1234", expect: "https://example.com:1234"},
+		"trailing slash":             {value: "example.com/", expect: "http://example.com:11434"},
+		"trailing slash port":        {value: "example.com:1234/", expect: "http://example.com:1234"},
+	}
+
+	for k, v := range testCases {
+		t.Run(k, func(t *testing.T) {
+			t.Setenv("OLLAMA_HOST", v.value)
+
+			client, err := ClientFromEnvironment()
+			if err != v.err {
+				t.Fatalf("expected %s, got %s", v.err, err)
+			}
+
+			if client.base.String() != v.expect {
+				t.Fatalf("expected %s, got %s", v.expect, client.base.String())
+			}
+		})
+	}
+}

api/types.go

@@ -1,9 +1,12 @@
 package api
 
 import (
+	"encoding/json"
 	"fmt"
+	"math"
 	"os"
-	"runtime"
+	"reflect"
+	"strings"
 	"time"
 )
 
@@ -28,26 +31,95 @@ func (e StatusError) Error() string {
 }
 
 type GenerateRequest struct {
-	Model   string `json:"model"`
-	Prompt  string `json:"prompt"`
-	Context []int  `json:"context,omitempty"`
+	Model    string `json:"model"`
+	Prompt   string `json:"prompt"`
+	System   string `json:"system"`
+	Template string `json:"template"`
+	Context  []int  `json:"context,omitempty"`
+	Stream   *bool  `json:"stream,omitempty"`
+	Raw      bool   `json:"raw,omitempty"`
+	Format   string `json:"format"`
 
-	Options `json:"options"`
+	Options map[string]interface{} `json:"options"`
+}
+
+// Options specfied in GenerateRequest, if you add a new option here add it to the API docs also
+type Options struct {
+	Runner
+
+	// Predict options used at runtime
+	NumKeep          int      `json:"num_keep,omitempty"`
+	Seed             int      `json:"seed,omitempty"`
+	NumPredict       int      `json:"num_predict,omitempty"`
+	TopK             int      `json:"top_k,omitempty"`
+	TopP             float32  `json:"top_p,omitempty"`
+	TFSZ             float32  `json:"tfs_z,omitempty"`
+	TypicalP         float32  `json:"typical_p,omitempty"`
+	RepeatLastN      int      `json:"repeat_last_n,omitempty"`
+	Temperature      float32  `json:"temperature,omitempty"`
+	RepeatPenalty    float32  `json:"repeat_penalty,omitempty"`
+	PresencePenalty  float32  `json:"presence_penalty,omitempty"`
+	FrequencyPenalty float32  `json:"frequency_penalty,omitempty"`
+	Mirostat         int      `json:"mirostat,omitempty"`
+	MirostatTau      float32  `json:"mirostat_tau,omitempty"`
+	MirostatEta      float32  `json:"mirostat_eta,omitempty"`
+	PenalizeNewline  bool     `json:"penalize_newline,omitempty"`
+	Stop             []string `json:"stop,omitempty"`
+}
+
+// Runner options which must be set when the model is loaded into memory
+type Runner struct {
+	UseNUMA            bool    `json:"numa,omitempty"`
+	NumCtx             int     `json:"num_ctx,omitempty"`
+	NumBatch           int     `json:"num_batch,omitempty"`
+	NumGQA             int     `json:"num_gqa,omitempty"`
+	NumGPU             int     `json:"num_gpu,omitempty"`
+	MainGPU            int     `json:"main_gpu,omitempty"`
+	LowVRAM            bool    `json:"low_vram,omitempty"`
+	F16KV              bool    `json:"f16_kv,omitempty"`
+	LogitsAll          bool    `json:"logits_all,omitempty"`
+	VocabOnly          bool    `json:"vocab_only,omitempty"`
+	UseMMap            bool    `json:"use_mmap,omitempty"`
+	UseMLock           bool    `json:"use_mlock,omitempty"`
+	EmbeddingOnly      bool    `json:"embedding_only,omitempty"`
+	RopeFrequencyBase  float32 `json:"rope_frequency_base,omitempty"`
+	RopeFrequencyScale float32 `json:"rope_frequency_scale,omitempty"`
+	NumThread          int     `json:"num_thread,omitempty"`
+}
+
+type EmbeddingRequest struct {
+	Model  string `json:"model"`
+	Prompt string `json:"prompt"`
+
+	Options map[string]interface{} `json:"options"`
+}
+
+type EmbeddingResponse struct {
+	Embedding []float64 `json:"embedding"`
 }
 
 type CreateRequest struct {
-	Name string `json:"name"`
-	Path string `json:"path"`
-}
-
-type CreateProgress struct {
-	Status string `json:"status"`
+	Name   string `json:"name"`
+	Path   string `json:"path"`
+	Stream *bool  `json:"stream,omitempty"`
 }
 
 type DeleteRequest struct {
 	Name string `json:"name"`
 }
 
+type ShowRequest struct {
+	Name string `json:"name"`
+}
+
+type ShowResponse struct {
+	License    string `json:"license,omitempty"`
+	Modelfile  string `json:"modelfile,omitempty"`
+	Parameters string `json:"parameters,omitempty"`
+	Template   string `json:"template,omitempty"`
+	System     string `json:"system,omitempty"`
+}
+
 type CopyRequest struct {
 	Source      string `json:"source"`
 	Destination string `json:"destination"`
@@ -58,13 +130,14 @@ type PullRequest struct {
 	Insecure bool   `json:"insecure,omitempty"`
 	Username string `json:"username"`
 	Password string `json:"password"`
+	Stream   *bool  `json:"stream,omitempty"`
 }
 
 type ProgressResponse struct {
 	Status    string `json:"status"`
 	Digest    string `json:"digest,omitempty"`
-	Total     int    `json:"total,omitempty"`
-	Completed int    `json:"completed,omitempty"`
+	Total     int64  `json:"total,omitempty"`
+	Completed int64  `json:"completed,omitempty"`
 }
 
 type PushRequest struct {
@@ -72,27 +145,34 @@ type PushRequest struct {
 	Insecure bool   `json:"insecure,omitempty"`
 	Username string `json:"username"`
 	Password string `json:"password"`
+	Stream   *bool  `json:"stream,omitempty"`
 }
 
 type ListResponse struct {
-	Models []ListResponseModel `json:"models"`
+	Models []ModelResponse `json:"models"`
 }
 
-type ListResponseModel struct {
+type ModelResponse struct {
 	Name       string    `json:"name"`
 	ModifiedAt time.Time `json:"modified_at"`
-	Size       int       `json:"size"`
+	Size       int64     `json:"size"`
+	Digest     string    `json:"digest"`
+}
+
+type TokenResponse struct {
+	Token string `json:"token"`
 }
 
 type GenerateResponse struct {
 	Model     string    `json:"model"`
 	CreatedAt time.Time `json:"created_at"`
-	Response  string    `json:"response,omitempty"`
+	Response  string    `json:"response"`
 
 	Done    bool  `json:"done"`
 	Context []int `json:"context,omitempty"`
 
 	TotalDuration      time.Duration `json:"total_duration,omitempty"`
+	LoadDuration       time.Duration `json:"load_duration,omitempty"`
 	PromptEvalCount    int           `json:"prompt_eval_count,omitempty"`
 	PromptEvalDuration time.Duration `json:"prompt_eval_duration,omitempty"`
 	EvalCount          int           `json:"eval_count,omitempty"`
@@ -104,6 +184,10 @@ func (r *GenerateResponse) Summary() {
 		fmt.Fprintf(os.Stderr, "total duration:       %v\n", r.TotalDuration)
 	}
 
+	if r.LoadDuration > 0 {
+		fmt.Fprintf(os.Stderr, "load duration:        %v\n", r.LoadDuration)
+	}
+
 	if r.PromptEvalCount > 0 {
 		fmt.Fprintf(os.Stderr, "prompt eval count:    %d token(s)\n", r.PromptEvalCount)
 	}
@@ -123,69 +207,155 @@ func (r *GenerateResponse) Summary() {
 	}
 }
 
-type Options struct {
-	Seed int `json:"seed,omitempty"`
+var ErrInvalidOpts = fmt.Errorf("invalid options")
 
-	// Backend options
-	UseNUMA bool `json:"numa,omitempty"`
+func (opts *Options) FromMap(m map[string]interface{}) error {
+	valueOpts := reflect.ValueOf(opts).Elem() // names of the fields in the options struct
+	typeOpts := reflect.TypeOf(opts).Elem()   // types of the fields in the options struct
 
-	// Model options
-	NumCtx        int  `json:"num_ctx,omitempty"`
-	NumBatch      int  `json:"num_batch,omitempty"`
-	NumGPU        int  `json:"num_gpu,omitempty"`
-	MainGPU       int  `json:"main_gpu,omitempty"`
-	LowVRAM       bool `json:"low_vram,omitempty"`
-	F16KV         bool `json:"f16_kv,omitempty"`
-	LogitsAll     bool `json:"logits_all,omitempty"`
-	VocabOnly     bool `json:"vocab_only,omitempty"`
-	UseMMap       bool `json:"use_mmap,omitempty"`
-	UseMLock      bool `json:"use_mlock,omitempty"`
-	EmbeddingOnly bool `json:"embedding_only,omitempty"`
+	// build map of json struct tags to their types
+	jsonOpts := make(map[string]reflect.StructField)
+	for _, field := range reflect.VisibleFields(typeOpts) {
+		jsonTag := strings.Split(field.Tag.Get("json"), ",")[0]
+		if jsonTag != "" {
+			jsonOpts[jsonTag] = field
+		}
+	}
 
-	// Predict options
-	RepeatLastN      int     `json:"repeat_last_n,omitempty"`
-	RepeatPenalty    float32 `json:"repeat_penalty,omitempty"`
-	FrequencyPenalty float32 `json:"frequency_penalty,omitempty"`
-	PresencePenalty  float32 `json:"presence_penalty,omitempty"`
-	Temperature      float32 `json:"temperature,omitempty"`
-	TopK             int     `json:"top_k,omitempty"`
-	TopP             float32 `json:"top_p,omitempty"`
-	TFSZ             float32 `json:"tfs_z,omitempty"`
-	TypicalP         float32 `json:"typical_p,omitempty"`
-	Mirostat         int     `json:"mirostat,omitempty"`
-	MirostatTau      float32 `json:"mirostat_tau,omitempty"`
-	MirostatEta      float32 `json:"mirostat_eta,omitempty"`
+	invalidOpts := []string{}
+	for key, val := range m {
+		if opt, ok := jsonOpts[key]; ok {
+			field := valueOpts.FieldByName(opt.Name)
+			if field.IsValid() && field.CanSet() {
+				if val == nil {
+					continue
+				}
 
-	NumThread int `json:"num_thread,omitempty"`
+				switch field.Kind() {
+				case reflect.Int:
+					switch t := val.(type) {
+					case int64:
+						field.SetInt(t)
+					case float64:
+						// when JSON unmarshals numbers, it uses float64, not int
+						field.SetInt(int64(t))
+					default:
+						return fmt.Errorf("option %q must be of type integer", key)
+					}
+				case reflect.Bool:
+					val, ok := val.(bool)
+					if !ok {
+						return fmt.Errorf("option %q must be of type boolean", key)
+					}
+					field.SetBool(val)
+				case reflect.Float32:
+					// JSON unmarshals to float64
+					val, ok := val.(float64)
+					if !ok {
+						return fmt.Errorf("option %q must be of type float32", key)
+					}
+					field.SetFloat(val)
+				case reflect.String:
+					val, ok := val.(string)
+					if !ok {
+						return fmt.Errorf("option %q must be of type string", key)
+					}
+					field.SetString(val)
+				case reflect.Slice:
+					// JSON unmarshals to []interface{}, not []string
+					val, ok := val.([]interface{})
+					if !ok {
+						return fmt.Errorf("option %q must be of type array", key)
+					}
+					// convert []interface{} to []string
+					slice := make([]string, len(val))
+					for i, item := range val {
+						str, ok := item.(string)
+						if !ok {
+							return fmt.Errorf("option %q must be of an array of strings", key)
+						}
+						slice[i] = str
+					}
+					field.Set(reflect.ValueOf(slice))
+				default:
+					return fmt.Errorf("unknown type loading config params: %v", field.Kind())
+				}
+			}
+		} else {
+			invalidOpts = append(invalidOpts, key)
+		}
+	}
+
+	if len(invalidOpts) > 0 {
+		return fmt.Errorf("%w: %v", ErrInvalidOpts, strings.Join(invalidOpts, ", "))
+	}
+	return nil
 }
 
 func DefaultOptions() Options {
 	return Options{
-		Seed: -1,
-
-		UseNUMA: false,
-
-		NumCtx:   2048,
-		NumBatch: 512,
-		NumGPU:   1,
-		LowVRAM:  false,
-		F16KV:    true,
-		UseMMap:  true,
-		UseMLock: false,
-
-		RepeatLastN:      512,
-		RepeatPenalty:    1.1,
-		FrequencyPenalty: 0.0,
-		PresencePenalty:  0.0,
+		// options set on request to runner
+		NumPredict:       -1,
+		NumKeep:          0,
 		Temperature:      0.8,
 		TopK:             40,
 		TopP:             0.9,
 		TFSZ:             1.0,
 		TypicalP:         1.0,
+		RepeatLastN:      64,
+		RepeatPenalty:    1.1,
+		PresencePenalty:  0.0,
+		FrequencyPenalty: 0.0,
 		Mirostat:         0,
 		MirostatTau:      5.0,
 		MirostatEta:      0.1,
+		PenalizeNewline:  true,
+		Seed:             -1,
 
-		NumThread: runtime.NumCPU(),
+		Runner: Runner{
+			// options set when the model is loaded
+			NumCtx:             2048,
+			RopeFrequencyBase:  10000.0,
+			RopeFrequencyScale: 1.0,
+			NumBatch:           512,
+			NumGPU:             -1, // -1 here indicates that NumGPU should be set dynamically
+			NumGQA:             1,
+			NumThread:          0, // let the runtime decide
+			LowVRAM:            false,
+			F16KV:              true,
+			UseMLock:           false,
+			UseMMap:            true,
+			UseNUMA:            false,
+			EmbeddingOnly:      true,
+		},
 	}
 }
+
+type Duration struct {
+	time.Duration
+}
+
+func (d *Duration) UnmarshalJSON(b []byte) (err error) {
+	var v any
+	if err := json.Unmarshal(b, &v); err != nil {
+		return err
+	}
+
+	d.Duration = 5 * time.Minute
+
+	switch t := v.(type) {
+	case float64:
+		if t < 0 {
+			t = math.MaxFloat64
+		}
+
+		d.Duration = time.Duration(t)
+	case string:
+		d.Duration, err = time.ParseDuration(t)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

app/README.md

@@ -1,7 +1,5 @@
 # Desktop
 
-_Note: the Ollama desktop app is a work in progress and is not ready yet for general use._
-
 This app builds upon Ollama to provide a desktop experience for running models.
 
 ## Developing
@@ -9,19 +7,15 @@ This app builds upon Ollama to provide a desktop experience for running models.
 First, build the `ollama` binary:
 
 ```
-make -C ..
+cd ..
+go build .
 ```
 
 Then run the desktop app with `npm start`:
 
 ```
+cd app
 npm install
 npm start
 ```
 
-## Coming soon
-
-- Browse the latest available models on Hugging Face and other sources
-- Keep track of previous conversations with models
-- Switch quickly between models
-- Connect to remote Ollama servers to run models

app/forge.config.ts

@@ -18,12 +18,15 @@ const config: ForgeConfig = {
     asar: true,
     icon: './assets/icon.icns',
     extraResource: [
-      '../ollama',
-      path.join(__dirname, './assets/ollama_icon_16x16Template.png'),
-      path.join(__dirname, './assets/ollama_icon_16x16Template@2x.png'),
-      path.join(__dirname, './assets/ollama_outline_icon_16x16Template.png'),
-      path.join(__dirname, './assets/ollama_outline_icon_16x16Template@2x.png'),
-      ...(process.platform === 'darwin' ? ['../llama/ggml-metal.metal'] : []),
+      '../dist/ollama',
+      path.join(__dirname, './assets/iconTemplate.png'),
+      path.join(__dirname, './assets/iconTemplate@2x.png'),
+      path.join(__dirname, './assets/iconUpdateTemplate.png'),
+      path.join(__dirname, './assets/iconUpdateTemplate@2x.png'),
+      path.join(__dirname, './assets/iconDarkTemplate.png'),
+      path.join(__dirname, './assets/iconDarkTemplate@2x.png'),
+      path.join(__dirname, './assets/iconDarkUpdateTemplate.png'),
+      path.join(__dirname, './assets/iconDarkUpdateTemplate@2x.png'),
     ],
     ...(process.env.SIGN
       ? {
@@ -38,19 +41,12 @@ const config: ForgeConfig = {
           },
         }
       : {}),
+    osxUniversal: {
+      x64ArchFiles: '**/ollama',
+    },
   },
   rebuildConfig: {},
   makers: [new MakerSquirrel({}), new MakerZIP({}, ['darwin'])],
-  publishers: [
-    new PublisherGithub({
-      repository: {
-        name: 'ollama',
-        owner: 'jmorganca',
-      },
-      draft: false,
-      prerelease: true,
-    }),
-  ],
   hooks: {
     readPackageJson: async (_, packageJson) => {
       return { ...packageJson, version: process.env.VERSION || packageJson.version }

app/package.json

@@ -6,10 +6,10 @@
   "main": ".webpack/main",
   "scripts": {
     "start": "electron-forge start",
-    "package": "electron-forge package",
-    "package:sign": "SIGN=1 electron-forge package",
-    "make": "electron-forge make",
-    "make:sign": "SIGN=1 electron-forge make",
+    "package": "electron-forge package --arch universal",
+    "package:sign": "SIGN=1 electron-forge package --arch universal",
+    "make": "electron-forge make --arch universal",
+    "make:sign": "SIGN=1 electron-forge make --arch universal",
     "publish": "SIGN=1 electron-forge publish",
     "lint": "eslint --ext .ts,.tsx .",
     "format": "prettier --check . --ignore-path .gitignore",
@@ -32,6 +32,7 @@
     "@electron-forge/plugin-auto-unpack-natives": "^6.2.1",
     "@electron-forge/plugin-webpack": "^6.2.1",
     "@electron-forge/publisher-github": "^6.2.1",
+    "@electron/universal": "^1.4.1",
     "@svgr/webpack": "^8.0.1",
     "@types/chmodr": "^1.0.0",
     "@types/node": "^20.4.0",
@@ -45,7 +46,7 @@
     "chmodr": "^1.2.0",
     "copy-webpack-plugin": "^11.0.0",
     "css-loader": "^6.8.1",
-    "electron": "25.2.0",
+    "electron": "25.9.2",
     "eslint": "^8.43.0",
     "eslint-plugin-import": "^2.27.5",
     "fork-ts-checker-webpack-plugin": "^7.3.0",

app/src/app.tsx

@@ -2,7 +2,7 @@ import { useState } from 'react'
 import copy from 'copy-to-clipboard'
 import { CheckIcon, DocumentDuplicateIcon } from '@heroicons/react/24/outline'
 import Store from 'electron-store'
-import { getCurrentWindow } from '@electron/remote'
+import { getCurrentWindow, app } from '@electron/remote'
 
 import { install } from './install'
 import OllamaIcon from './ollama.svg'
@@ -51,10 +51,15 @@ export default function () {
               <div className='mx-auto'>
                 <button
                   onClick={async () => {
-                    await install()
-                    getCurrentWindow().show()
-                    getCurrentWindow().focus()
-                    setStep(Step.FINISH)
+                    try {
+                      await install()
+                      setStep(Step.FINISH)
+                    } catch (e) {
+                      console.error('could not install: ', e)
+                    } finally {
+                      getCurrentWindow().show()
+                      getCurrentWindow().focus()
+                    }
                   }}
                   className='no-drag rounded-dm mx-auto w-[60%] rounded-md bg-black px-4 py-2 text-sm text-white hover:brightness-110'
                 >

app/src/index.ts

@@ -1,17 +1,21 @@
-import { spawn } from 'child_process'
-import { app, autoUpdater, dialog, Tray, Menu, BrowserWindow, nativeTheme } from 'electron'
+import { spawn, ChildProcess } from 'child_process'
+import { app, autoUpdater, dialog, Tray, Menu, BrowserWindow, MenuItemConstructorOptions, nativeTheme } from 'electron'
 import Store from 'electron-store'
 import winston from 'winston'
 import 'winston-daily-rotate-file'
 import * as path from 'path'
 
-import { analytics, id } from './telemetry'
+import { v4 as uuidv4 } from 'uuid'
 import { installed } from './install'
 
 require('@electron/remote/main').initialize()
 
+if (require('electron-squirrel-startup')) {
+  app.quit()
+}
+
 const store = new Store()
-let tray: Tray | null = null
+
 let welcomeWindow: BrowserWindow | null = null
 
 declare const MAIN_WINDOW_WEBPACK_ENTRY: string
@@ -28,10 +32,30 @@ const logger = winston.createLogger({
   format: winston.format.printf(info => info.message),
 })
 
-const SingleInstanceLock = app.requestSingleInstanceLock()
-if (!SingleInstanceLock) {
-  app.quit()
-}
+app.on('ready', () => {
+  const gotTheLock = app.requestSingleInstanceLock()
+  if (!gotTheLock) {
+    app.exit(0)
+    return
+  }
+
+  app.on('second-instance', () => {
+    if (app.hasSingleInstanceLock()) {
+      app.releaseSingleInstanceLock()
+    }
+
+    if (proc) {
+      proc.off('exit', restart)
+      proc.kill()
+    }
+
+    app.exit(0)
+  })
+
+  app.focus({ steal: true })
+
+  init()
+})
 
 function firstRunWindow() {
   // Create the browser window.
@@ -47,65 +71,74 @@ function firstRunWindow() {
       nodeIntegration: true,
       contextIsolation: false,
     },
-    alwaysOnTop: true,
   })
 
   require('@electron/remote/main').enable(welcomeWindow.webContents)
 
-  // and load the index.html of the app.
   welcomeWindow.loadURL(MAIN_WINDOW_WEBPACK_ENTRY)
-
   welcomeWindow.on('ready-to-show', () => welcomeWindow.show())
-
-  // for debugging
-  // welcomeWindow.webContents.openDevTools()
-
-  if (process.platform === 'darwin') {
-    app.dock.hide()
-  }
-}
-
-function createSystemtray() {
-  let iconPath = nativeTheme.shouldUseDarkColors
-    ? path.join(__dirname, '..', '..', 'assets', 'ollama_icon_16x16Template.png')
-    : path.join(__dirname, '..', '..', 'assets', 'ollama_outline_icon_16x16Template.png')
-
-  if (app.isPackaged) {
-    iconPath = nativeTheme.shouldUseDarkColors
-      ? path.join(process.resourcesPath, 'ollama_icon_16x16Template.png')
-      : path.join(process.resourcesPath, 'ollama_outline_icon_16x16Template.png')
-  }
-
-  tray = new Tray(iconPath)
-
-  nativeTheme.on('updated', function theThemeHasChanged() {
-    if (nativeTheme.shouldUseDarkColors) {
-      app.isPackaged
-        ? tray.setImage(path.join(process.resourcesPath, 'ollama_icon_16x16Template.png'))
-        : tray.setImage(path.join(__dirname, '..', '..', 'assets', 'ollama_icon_16x16Template.png'))
-    } else {
-      app.isPackaged
-        ? tray.setImage(path.join(process.resourcesPath, 'ollama_outline_icon_16x16Template.png'))
-        : tray.setImage(path.join(__dirname, '..', '..', 'assets', 'ollama_outline_icon_16x16Template.png'))
+  welcomeWindow.on('closed', () => {
+    if (process.platform === 'darwin') {
+      app.dock.hide()
     }
   })
-
-  const contextMenu = Menu.buildFromTemplate([{ role: 'quit', label: 'Quit Ollama', accelerator: 'Command+Q' }])
-
-  tray.setContextMenu(contextMenu)
-  tray.setToolTip('Ollama')
 }
 
-if (require('electron-squirrel-startup')) {
-  app.quit()
+let tray: Tray | null = null
+let updateAvailable = false
+const assetPath = app.isPackaged ? process.resourcesPath : path.join(__dirname, '..', '..', 'assets')
+
+function trayIconPath() {
+  return nativeTheme.shouldUseDarkColors
+    ? updateAvailable
+      ? path.join(assetPath, 'iconDarkUpdateTemplate.png')
+      : path.join(assetPath, 'iconDarkTemplate.png')
+    : updateAvailable
+    ? path.join(assetPath, 'iconUpdateTemplate.png')
+    : path.join(assetPath, 'iconTemplate.png')
 }
 
+function updateTrayIcon() {
+  if (tray) {
+    tray.setImage(trayIconPath())
+  }
+}
+
+function updateTray() {
+  const updateItems: MenuItemConstructorOptions[] = [
+    { label: 'An update is available', enabled: false },
+    {
+      label: 'Restart to update',
+      click: () => autoUpdater.quitAndInstall(),
+    },
+    { type: 'separator' },
+  ]
+
+  const menu = Menu.buildFromTemplate([
+    ...(updateAvailable ? updateItems : []),
+    { role: 'quit', label: 'Quit Ollama', accelerator: 'Command+Q' },
+  ])
+
+  if (!tray) {
+    tray = new Tray(trayIconPath())
+  }
+
+  tray.setToolTip(updateAvailable ? 'An update is available' : 'Ollama')
+  tray.setContextMenu(menu)
+  tray.setImage(trayIconPath())
+
+  nativeTheme.off('updated', updateTrayIcon)
+  nativeTheme.on('updated', updateTrayIcon)
+}
+
+let proc: ChildProcess = null
+
 function server() {
   const binary = app.isPackaged
     ? path.join(process.resourcesPath, 'ollama')
     : path.resolve(process.cwd(), '..', 'ollama')
 
-  const proc = spawn(binary, ['serve'])
+  proc = spawn(binary, ['serve'])
 
   proc.stdout.on('data', data => {
     logger.info(data.toString().trim())
@@ -115,23 +148,75 @@ function server() {
     logger.error(data.toString().trim())
   })
 
-  function restart() {
-    setTimeout(server, 3000)
+  proc.on('exit', restart)
+}
+
+function restart() {
+  setTimeout(server, 1000)
+}
+
+app.on('before-quit', () => {
+  if (proc) {
+    proc.off('exit', restart)
+    proc.kill('SIGINT') // send SIGINT signal to the server, which also stops any loaded llms
+  }
+})
+
+const updateURL = `https://ollama.ai/api/update?os=${process.platform}&arch=${
+  process.arch
+}&version=${app.getVersion()}&id=${id()}`
+
+let latest = ''
+async function isNewReleaseAvailable() {
+  try {
+    const response = await fetch(updateURL)
+
+    if (!response.ok) {
+      return false
+    }
+
+    if (response.status === 204) {
+      return false
+    }
+
+    const data = await response.json()
+
+    const url = data?.url
+    if (!url) {
+      return false
+    }
+
+    if (latest === url) {
+      return false
+    }
+
+    latest = url
+
+    return true
+  } catch (error) {
+    logger.error(`update check failed - ${error}`)
+    return false
+  }
+}
+
+async function checkUpdate() {
+  const available = await isNewReleaseAvailable()
+  if (available) {
+    logger.info('checking for update')
+    autoUpdater.checkForUpdates()
+  }
+}
+
+function init() {
+  if (app.isPackaged) {
+    checkUpdate()
+    setInterval(() => {
+      checkUpdate()
+    }, 60 * 60 * 1000)
   }
 
-  proc.on('exit', restart)
+  updateTray()
 
-  app.on('before-quit', () => {
-    proc.off('exit', restart)
-    proc.kill()
-  })
-}
-
-if (process.platform === 'darwin') {
-  app.dock.hide()
-}
-
-app.on('ready', () => {
   if (process.platform === 'darwin') {
     if (app.isPackaged) {
       if (!app.isInApplicationsFolder()) {
@@ -167,10 +252,13 @@ app.on('ready', () => {
     }
   }
 
-  createSystemtray()
   server()
 
   if (store.get('first-time-run') && installed()) {
+    if (process.platform === 'darwin') {
+      app.dock.hide()
+    }
+
     app.setLoginItemSettings({ openAtLogin: app.getLoginItemSettings().openAtLogin })
     return
   }
@@ -178,7 +266,7 @@ app.on('ready', () => {
   // This is the first run or the CLI is no longer installed
   app.setLoginItemSettings({ openAtLogin: true })
   firstRunWindow()
-})
+}
 
 // Quit when all windows are closed, except on macOS. There, it's common
 // for applications and their menu bar to stay active until the user quits
@@ -189,45 +277,26 @@ app.on('window-all-closed', () => {
   }
 })
 
-// In this file you can include the rest of your app's specific main process
-// code. You can also put them in separate files and import them here.
-autoUpdater.setFeedURL({
-  url: `https://ollama.ai/api/update?os=${process.platform}&arch=${process.arch}&version=${app.getVersion()}`,
-})
+function id(): string {
+  const id = store.get('id') as string
 
-async function heartbeat() {
-  analytics.track({
-    anonymousId: id(),
-    event: 'heartbeat',
-    properties: {
-      version: app.getVersion(),
-    },
-  })
+  if (id) {
+    return id
+  }
+
+  const uuid = uuidv4()
+  store.set('id', uuid)
+  return uuid
 }
 
-if (app.isPackaged) {
-  heartbeat()
-  autoUpdater.checkForUpdates()
-  setInterval(() => {
-    heartbeat()
-    autoUpdater.checkForUpdates()
-  }, 60 * 60 * 1000)
-}
+autoUpdater.setFeedURL({ url: updateURL })
 
 autoUpdater.on('error', e => {
   logger.error(`update check failed - ${e.message}`)
+  console.error(`update check failed - ${e.message}`)
 })
 
-autoUpdater.on('update-downloaded', (event, releaseNotes, releaseName) => {
-  dialog
-    .showMessageBox({
-      type: 'info',
-      buttons: ['Restart Now', 'Later'],
-      title: 'New update available',
-      message: process.platform === 'win32' ? releaseNotes : releaseName,
-      detail: 'A new version of Ollama is available. Restart to apply the update.',
-    })
-    .then(returnValue => {
-      if (returnValue.response === 0) autoUpdater.quitAndInstall()
-    })
+autoUpdater.on('update-downloaded', () => {
+  updateAvailable = true
+  updateTray()
 })

app/src/install.ts

@@ -15,12 +15,7 @@ export function installed() {
 export async function install() {
   const command = `do shell script "mkdir -p ${path.dirname(
     symlinkPath
-  )} && ln -F -s ${ollama} ${symlinkPath}" with administrator privileges`
+  )} && ln -F -s \\"${ollama}\\" \\"${symlinkPath}\\"" with administrator privileges`
 
-  try {
-    await exec(`osascript -e '${command}'`)
-  } catch (error) {
-    console.error(`cli: failed to install cli: ${error.message}`)
-    return
-  }
+  await exec(`osascript -e '${command}'`)
 }

app/src/telemetry.ts

@@ -1,19 +0,0 @@
-import { Analytics } from '@segment/analytics-node'
-import { v4 as uuidv4 } from 'uuid'
-import Store from 'electron-store'
-
-const store = new Store()
-
-export const analytics = new Analytics({ writeKey: process.env.TELEMETRY_WRITE_KEY || '<empty>' })
-
-export function id(): string {
-  const id = store.get('id') as string
-
-  if (id) {
-    return id
-  }
-
-  const uuid = uuidv4()
-  store.set('id', uuid)
-  return uuid
-}

docs/README.md

@@ -0,0 +1,6 @@
+# Documentation
+
+- [Modelfile](./modelfile.md)
+- [How to develop Ollama](./development.md)
+- [API](./api.md)
+- [Tutorials](./tutorials.md)

docs/api.md

@@ -0,0 +1,617 @@
+# API
+
+## Endpoints
+
+- [Generate a completion](#generate-a-completion)
+- [Create a Model](#create-a-model)
+- [List Local Models](#list-local-models)
+- [Show Model Information](#show-model-information)
+- [Copy a Model](#copy-a-model)
+- [Delete a Model](#delete-a-model)
+- [Pull a Model](#pull-a-model)
+- [Push a Model](#push-a-model)
+- [Generate Embeddings](#generate-embeddings)
+
+## Conventions
+
+### Model names
+
+Model names follow a `model:tag` format. Some examples are `orca-mini:3b-q4_1` and `llama2:70b`. The tag is optional and, if not provided, will default to `latest`. The tag is used to identify a specific version.
+
+### Durations
+
+All durations are returned in nanoseconds.
+
+### Streaming responses
+
+Certain endpoints stream responses as JSON objects delineated with the newline (`\n`) character.
+
+## Generate a completion
+
+```shell
+POST /api/generate
+```
+
+Generate a response for a given prompt with a provided model. This is a streaming endpoint, so will be a series of responses. The final response object will include statistics and additional data from the request.
+
+### Parameters
+
+- `model`: (required) the [model name](#model-names)
+- `prompt`: the prompt to generate a response for
+- `format`: the format to return a response in. Currently the only accepted value is `json`
+
+Advanced parameters (optional):
+
+- `options`: additional model parameters listed in the documentation for the [Modelfile](./modelfile.md#valid-parameters-and-values) such as `temperature`
+- `system`: system prompt to (overrides what is defined in the `Modelfile`)
+- `template`: the full prompt or prompt template (overrides what is defined in the `Modelfile`)
+- `context`: the context parameter returned from a previous request to `/generate`, this can be used to keep a short conversational memory
+- `stream`: if `false` the response will be returned as a single response object, rather than a stream of objects
+- `raw`: if `true` no formatting will be applied to the prompt and no context will be returned. You may choose to use the `raw` parameter if you are specifying a full templated prompt in your request to the API, and are managing history yourself.
+
+### JSON mode
+
+Enable JSON mode by setting the `format` parameter to `json` and specifying the model should use JSON in the `prompt`. This will structure the response as valid JSON. See the JSON mode [example](#request-json-mode) below.
+
+### Examples
+
+#### Request
+
+```shell
+curl -X POST http://localhost:11434/api/generate -d '{
+  "model": "llama2",
+  "prompt": "Why is the sky blue?"
+}'
+```
+
+#### Response
+
+A stream of JSON objects is returned:
+
+```json
+{
+  "model": "llama2",
+  "created_at": "2023-08-04T08:52:19.385406455-07:00",
+  "response": "The",
+  "done": false
+}
+```
+
+The final response in the stream also includes additional data about the generation:
+
+- `total_duration`: time spent generating the response
+- `load_duration`: time spent in nanoseconds loading the model
+- `sample_count`: number of samples generated
+- `sample_duration`: time spent generating samples
+- `prompt_eval_count`: number of tokens in the prompt
+- `prompt_eval_duration`: time spent in nanoseconds evaluating the prompt
+- `eval_count`: number of tokens the response
+- `eval_duration`: time in nanoseconds spent generating the response
+- `context`: an encoding of the conversation used in this response, this can be sent in the next request to keep a conversational memory
+- `response`: empty if the response was streamed, if not streamed, this will contain the full response
+
+To calculate how fast the response is generated in tokens per second (token/s), divide `eval_count` / `eval_duration`.
+
+```json
+{
+  "model": "llama2",
+  "created_at": "2023-08-04T19:22:45.499127Z",
+  "response": "",
+  "context": [1, 2, 3],
+  "done": true,
+  "total_duration": 5589157167,
+  "load_duration": 3013701500,
+  "sample_count": 114,
+  "sample_duration": 81442000,
+  "prompt_eval_count": 46,
+  "prompt_eval_duration": 1160282000,
+  "eval_count": 113,
+  "eval_duration": 1325948000
+}
+```
+
+#### Request (No streaming)
+
+```shell
+curl -X POST http://localhost:11434/api/generate -d '{
+  "model": "llama2:7b",
+  "prompt": "Why is the sky blue?",
+  "stream": false
+}'
+```
+
+#### Response
+
+If `stream` is set to `false`, the response will be a single JSON object:
+
+```json
+{
+  "model": "llama2:7b",
+  "created_at": "2023-08-04T19:22:45.499127Z",
+  "response": "The sky is blue because it is the color of the sky.",
+  "context": [1, 2, 3],
+  "done": true,
+  "total_duration": 5589157167,
+  "load_duration": 3013701500,
+  "sample_count": 114,
+  "sample_duration": 81442000,
+  "prompt_eval_count": 46,
+  "prompt_eval_duration": 1160282000,
+  "eval_count": 13,
+  "eval_duration": 1325948000
+}
+```
+
+#### Request (Raw mode)
+
+In some cases you may wish to bypass the templating system and provide a full prompt. In this case, you can use the `raw` parameter to disable formatting and context.
+
+```shell
+curl -X POST http://localhost:11434/api/generate -d '{
+  "model": "mistral",
+  "prompt": "[INST] why is the sky blue? [/INST]",
+  "raw": true,
+  "stream": false
+}'
+```
+
+#### Response
+
+```json
+{
+  "model": "mistral",
+  "created_at": "2023-11-03T15:36:02.583064Z",
+  "response": " The sky appears blue because of a phenomenon called Rayleigh scattering.",
+  "done": true,
+  "total_duration": 14648695333,
+  "load_duration": 3302671417,
+  "prompt_eval_count": 14,
+  "prompt_eval_duration": 286243000,
+  "eval_count": 129,
+  "eval_duration": 10931424000
+}
+```
+
+#### Request (JSON mode)
+
+```shell
+curl -X POST http://localhost:11434/api/generate -d '{
+  "model": "llama2",
+  "prompt": "What color is the sky at different times of the day? Respond using JSON",
+  "format": "json",
+  "stream": false
+}'
+```
+
+#### Response
+
+```json
+{
+  "model": "llama2",
+  "created_at": "2023-11-09T21:07:55.186497Z",
+  "response": "{\n\"morning\": {\n\"color\": \"blue\"\n},\n\"noon\": {\n\"color\": \"blue-gray\"\n},\n\"afternoon\": {\n\"color\": \"warm gray\"\n},\n\"evening\": {\n\"color\": \"orange\"\n}\n}\n",
+  "done": true,
+  "total_duration": 4661289125,
+  "load_duration": 1714434500,
+  "prompt_eval_count": 36,
+  "prompt_eval_duration": 264132000,
+  "eval_count": 75,
+  "eval_duration": 2112149000
+}
+```
+
+The value of `response` will be a string containing JSON similar to:
+
+```json
+{
+  "morning": {
+    "color": "blue"
+  },
+  "noon": {
+    "color": "blue-gray"
+  },
+  "afternoon": {
+    "color": "warm gray"
+  },
+  "evening": {
+    "color": "orange"
+  }
+}
+```
+
+#### Request (With options)
+
+If you want to set custom options for the model at runtime rather than in the Modelfile, you can do so with the `options` parameter. This example sets every available option, but you can set any of them individually and omit the ones you do not want to override.
+
+```shell
+curl -X POST http://localhost:11434/api/generate -d '{
+  "model": "llama2:7b",
+  "prompt": "Why is the sky blue?",
+  "stream": false,
+  "options": {
+    "num_keep": 5,
+    "seed": 42,
+    "num_predict": 100,
+    "top_k": 20,
+    "top_p": 0.9,
+    "tfs_z": 0.5,
+    "typical_p": 0.7,
+    "repeat_last_n": 33,
+    "temperature": 0.8,
+    "repeat_penalty": 1.2,
+    "presence_penalty": 1.5,
+    "frequency_penalty": 1.0,
+    "mirostat": 1,
+    "mirostat_tau": 0.8,
+    "mirostat_eta": 0.6,
+    "penalize_newline": true,
+    "stop": ["\n", "user:"],
+    "numa": false,
+    "num_ctx": 4,
+    "num_batch": 2,
+    "num_gqa": 1,
+    "num_gpu": 1,
+    "main_gpu": 0,
+    "low_vram": false,
+    "f16_kv": true,
+    "logits_all": false,
+    "vocab_only": false,
+    "use_mmap": true,
+    "use_mlock": false,
+    "embedding_only": false,
+    "rope_frequency_base": 1.1,
+    "rope_frequency_scale": 0.8,
+    "num_thread": 8
+    }
+}'
+```
+
+#### Response
+
+```json
+{
+  "model": "llama2:7b",
+  "created_at": "2023-08-04T19:22:45.499127Z",
+  "response": "The sky is blue because it is the color of the sky.",
+  "context": [1, 2, 3],
+  "done": true,
+  "total_duration": 5589157167,
+  "load_duration": 3013701500,
+  "sample_count": 114,
+  "sample_duration": 81442000,
+  "prompt_eval_count": 46,
+  "prompt_eval_duration": 1160282000,
+  "eval_count": 13,
+  "eval_duration": 1325948000
+}
+```
+
+## Create a Model
+
+```shell
+POST /api/create
+```
+
+Create a model from a [`Modelfile`](./modelfile.md)
+
+### Parameters
+
+- `name`: name of the model to create
+- `path`: path to the Modelfile
+- `stream`: (optional) if `false` the response will be returned as a single response object, rather than a stream of objects
+
+### Examples
+
+#### Request
+
+```shell
+curl -X POST http://localhost:11434/api/create -d '{
+  "name": "mario",
+  "path": "~/Modelfile"
+}'
+```
+
+#### Response
+
+A stream of JSON objects. When finished, `status` is `success`.
+
+```json
+{
+  "status": "parsing modelfile"
+}
+```
+
+## List Local Models
+
+```shell
+GET /api/tags
+```
+
+List models that are available locally.
+
+### Examples
+
+#### Request
+
+```shell
+curl http://localhost:11434/api/tags
+```
+
+#### Response
+
+A single JSON object will be returned.
+
+```json
+{
+  "models": [
+    {
+      "name": "llama2:7b",
+      "modified_at": "2023-08-02T17:02:23.713454393-07:00",
+      "size": 3791730596
+    },
+    {
+      "name": "llama2:13b",
+      "modified_at": "2023-08-08T12:08:38.093596297-07:00",
+      "size": 7323310500
+    }
+  ]
+}
+```
+
+## Show Model Information
+
+```shell
+POST /api/show
+```
+
+Show details about a model including modelfile, template, parameters, license, and system prompt.
+
+### Parameters
+
+- `name`: name of the model to show
+
+### Examples
+
+#### Request
+
+```shell
+curl http://localhost:11434/api/show -d '{
+  "name": "llama2:7b"
+}'
+```
+
+#### Response
+
+```json
+{
+  "license": "<contents of license block>",
+  "modelfile": "# Modelfile generated by \"ollama show\"\n# To build a new Modelfile based on this one, replace the FROM line with:\n# FROM llama2:latest\n\nFROM /Users/username/.ollama/models/blobs/sha256:8daa9615cce30c259a9555b1cc250d461d1bc69980a274b44d7eda0be78076d8\nTEMPLATE \"\"\"[INST] {{ if and .First .System }}<<SYS>>{{ .System }}<</SYS>>\n\n{{ end }}{{ .Prompt }} [/INST] \"\"\"\nSYSTEM \"\"\"\"\"\"\nPARAMETER stop [INST]\nPARAMETER stop [/INST]\nPARAMETER stop <<SYS>>\nPARAMETER stop <</SYS>>\n",
+  "parameters": "stop                           [INST]\nstop                           [/INST]\nstop                           <<SYS>>\nstop                           <</SYS>>",
+  "template": "[INST] {{ if and .First .System }}<<SYS>>{{ .System }}<</SYS>>\n\n{{ end }}{{ .Prompt }} [/INST] "
+}
+```
+
+## Copy a Model
+
+```shell
+POST /api/copy
+```
+
+Copy a model. Creates a model with another name from an existing model.
+
+### Examples
+
+#### Request
+
+```shell
+curl http://localhost:11434/api/copy -d '{
+  "source": "llama2:7b",
+  "destination": "llama2-backup"
+}'
+```
+
+#### Response
+
+The only response is a 200 OK if successful.
+
+## Delete a Model
+
+```shell
+DELETE /api/delete
+```
+
+Delete a model and its data.
+
+### Parameters
+
+- `name`: model name to delete
+
+### Examples
+
+#### Request
+
+```shell
+curl -X DELETE http://localhost:11434/api/delete -d '{
+  "name": "llama2:13b"
+}'
+```
+
+#### Response
+
+If successful, the only response is a 200 OK.
+
+## Pull a Model
+
+```shell
+POST /api/pull
+```
+
+Download a model from the ollama library. Cancelled pulls are resumed from where they left off, and multiple calls will share the same download progress.
+
+### Parameters
+
+- `name`: name of the model to pull
+- `insecure`: (optional) allow insecure connections to the library. Only use this if you are pulling from your own library during development.
+- `stream`: (optional) if `false` the response will be returned as a single response object, rather than a stream of objects
+
+### Examples
+
+#### Request
+
+```shell
+curl -X POST http://localhost:11434/api/pull -d '{
+  "name": "llama2:7b"
+}'
+```
+
+#### Response
+
+If `stream` is not specified, or set to `true`, a stream of JSON objects is returned:
+
+The first object is the manifest:
+
+```json
+{
+  "status": "pulling manifest"
+}
+```
+
+Then there is a series of downloading responses. Until any of the download is completed, the `completed` key may not be included. The number of files to be downloaded depends on the number of layers specified in the manifest.
+
+```json
+{
+  "status": "downloading digestname",
+  "digest": "digestname",
+  "total": 2142590208,
+  "completed": 241970
+}
+```
+
+After all the files are downloaded, the final responses are:
+
+```json
+{
+    "status": "verifying sha256 digest"
+}
+{
+    "status": "writing manifest"
+}
+{
+    "status": "removing any unused layers"
+}
+{
+    "status": "success"
+}
+```
+
+if `stream` is set to false, then the response is a single JSON object:
+
+```json
+{
+  "status": "success"
+}
+```
+
+## Push a Model
+
+```shell
+POST /api/push
+```
+
+Upload a model to a model library. Requires registering for ollama.ai and adding a public key first.
+
+### Parameters
+
+- `name`: name of the model to push in the form of `<namespace>/<model>:<tag>`
+- `insecure`: (optional) allow insecure connections to the library. Only use this if you are pushing to your library during development.
+- `stream`: (optional) if `false` the response will be returned as a single response object, rather than a stream of objects
+
+### Examples
+
+#### Request
+
+```shell
+curl -X POST http://localhost:11434/api/push -d '{
+  "name": "mattw/pygmalion:latest"
+}'
+```
+
+#### Response
+
+If `stream` is not specified, or set to `true`, a stream of JSON objects is returned:
+
+```json
+{ "status": "retrieving manifest" }
+```
+
+and then:
+
+```json
+{
+  "status": "starting upload",
+  "digest": "sha256:bc07c81de745696fdf5afca05e065818a8149fb0c77266fb584d9b2cba3711ab",
+  "total": 1928429856
+}
+```
+
+Then there is a series of uploading responses:
+
+```json
+{
+  "status": "starting upload",
+  "digest": "sha256:bc07c81de745696fdf5afca05e065818a8149fb0c77266fb584d9b2cba3711ab",
+  "total": 1928429856
+}
+```
+
+Finally, when the upload is complete:
+
+```json
+{"status":"pushing manifest"}
+{"status":"success"}
+```
+
+If `stream` is set to `false`, then the response is a single JSON object:
+
+```json
+{ "status": "success" }
+```
+
+## Generate Embeddings
+
+```shell
+POST /api/embeddings
+```
+
+Generate embeddings from a model
+
+### Parameters
+
+- `model`: name of model to generate embeddings from
+- `prompt`: text to generate embeddings for
+
+Advanced parameters:
+
+- `options`: additional model parameters listed in the documentation for the [Modelfile](./modelfile.md#valid-parameters-and-values) such as `temperature`
+
+### Examples
+
+#### Request
+
+```shell
+curl -X POST http://localhost:11434/api/embeddings -d '{
+  "model": "llama2:7b",
+  "prompt": "Here is an article about llamas..."
+}'
+```
+
+#### Response
+
+```json
+{
+  "embedding": [
+    0.5670403838157654, 0.009260174818336964, 0.23178744316101074, -0.2916173040866852, -0.8924556970596313,
+    0.8785552978515625, -0.34576427936553955, 0.5742510557174683, -0.04222835972905159, -0.137906014919281
+  ]
+}
+```

docs/development.md

@@ -1,46 +1,39 @@
 # Development
 
+- Install cmake or (optionally, required tools for GPUs)
+- run `go generate ./...`
+- run `go build .`
+
 Install required tools:
 
-```
-brew install go
+- cmake version 3.24 or higher
+- go version 1.20 or higher
+- gcc version 11.4.0 or higher
+
+```bash
+brew install go cmake gcc
 ```
 
-Enable CGO:
+Get the required libraries:
 
-```
-export CGO_ENABLED=1
+```bash
+go generate ./...
 ```
 
 Then build ollama:
 
-```
+```bash
 go build .
 ```
 
 Now you can run `ollama`:
 
-```
+```bash
 ./ollama
 ```
 
-## Releasing
-
-To release a new version of Ollama you'll need to set some environment variables:
-
-* `GITHUB_TOKEN`: your GitHub token
-* `APPLE_IDENTITY`: the Apple signing identity (macOS only)
-* `APPLE_ID`: your Apple ID
-* `APPLE_PASSWORD`: your Apple ID app-specific password
-* `APPLE_TEAM_ID`: the Apple team ID for the signing identity
-* `TELEMETRY_WRITE_KEY`: segment write key for telemetry
-
-Then run the publish script with the target version:
-
-```
-VERSION=0.0.2 ./scripts/publish.sh
-```
-
-
-
+## Building on Linux with GPU support
 
+- Install cmake and nvidia-cuda-toolkit
+- run `go generate ./...`
+- run `go build .`

docs/faq.md

@@ -0,0 +1,98 @@
+# FAQ
+
+## How can I view the logs?
+
+On macOS:
+
+```
+cat ~/.ollama/logs/server.log
+```
+
+On Linux:
+
+```
+journalctl -u ollama
+```
+
+If you're running `ollama serve` directly, the logs will be printed to the console.
+
+## How can I expose Ollama on my network?
+
+Ollama binds to 127.0.0.1 port 11434 by default. Change the bind address with the `OLLAMA_HOST` environment variable.
+
+On macOS:
+
+```bash
+OLLAMA_HOST=0.0.0.0:11435 ollama serve
+```
+
+On Linux:
+
+Create a `systemd` drop-in directory and set `Environment=OLLAMA_HOST`
+
+```bash
+mkdir -p /etc/systemd/system/ollama.service.d
+echo "[Service]" >>/etc/systemd/system/ollama.service.d/environment.conf
+```
+
+```bash
+echo "Environment=OLLAMA_HOST=0.0.0.0:11434" >>/etc/systemd/system/ollama.service.d/environment.conf
+```
+
+Reload `systemd` and restart Ollama:
+
+```bash
+systemctl daemon-reload
+systemctl restart ollama
+```
+
+## How can I allow additional web origins to access Ollama?
+
+Ollama allows cross origin requests from `127.0.0.1` and `0.0.0.0` by default. Add additional origins with the `OLLAMA_ORIGINS` environment variable:
+
+On macOS:
+
+```bash
+OLLAMA_ORIGINS=http://192.168.1.1:*,https://example.com ollama serve
+```
+
+On Linux:
+
+```bash
+echo "Environment=OLLAMA_ORIGINS=http://129.168.1.1:*,https://example.com" >>/etc/systemd/system/ollama.service.d/environment.conf
+```
+
+Reload `systemd` and restart Ollama:
+
+```bash
+systemctl daemon-reload
+systemctl restart ollama
+```
+
+## Where are models stored?
+
+- macOS: Raw model data is stored under `~/.ollama/models`.
+- Linux: Raw model data is stored under `/usr/share/ollama/.ollama/models`
+
+
+
+Below the models directory you will find a structure similar to the following:
+
+```shell
+.
+├── blobs
+└── manifests
+   └── registry.ollama.ai
+      ├── f0rodo
+      ├── library
+      ├── mattw
+      └── saikatkumardey
+```
+
+There is a `manifests/registry.ollama.ai/namespace` path. In example above, the user has downloaded models from the official `library`, `f0rodo`, `mattw`, and `saikatkumardey` namespaces. Within each of those directories, you will find directories for each of the models downloaded. And in there you will find a file name representing each tag. Each tag file is the manifest for the model.  
+
+The manifest lists all the layers used in this model. You will see a `media type` for each layer, along with a digest. That digest corresponds with a file in the `models/blobs directory`.
+
+### How can I change where Ollama stores models?
+
+To modify where models are stored, you can use the `OLLAMA_MODELS` environment variable. Note that on Linux this means defining `OLLAMA_MODELS` in a drop-in `/etc/systemd/system/ollama.service.d` service file, reloading systemd, and restarting the ollama service.

docs/import.md

@@ -0,0 +1,198 @@
+# Import a model
+
+This guide walks through importing a GGUF, PyTorch or Safetensors model.
+
+## Importing (GGUF)
+
+### Step 1: Write a `Modelfile`
+
+Start by creating a `Modelfile`. This file is the blueprint for your model, specifying weights, parameters, prompt templates and more.
+
+```
+FROM ./mistral-7b-v0.1.Q4_0.gguf
+```
+
+(Optional) many chat models require a prompt template in order to answer correctly. A default prompt template can be specified with the `TEMPLATE` instruction in the `Modelfile`:
+
+```
+FROM ./q4_0.bin
+TEMPLATE "[INST] {{ .Prompt }} [/INST]"
+```
+
+### Step 2: Create the Ollama model
+
+Finally, create a model from your `Modelfile`:
+
+```
+ollama create example -f Modelfile
+```
+
+### Step 3: Run your model
+
+Next, test the model with `ollama run`:
+
+```
+ollama run example "What is your favourite condiment?"
+```
+
+## Importing (PyTorch & Safetensors)
+
+### Supported models
+
+Ollama supports a set of model architectures, with support for more coming soon:
+
+- Llama & Mistral
+- Falcon & RW
+- GPT-NeoX
+- BigCode
+
+To view a model's architecture, check the `config.json` file in its HuggingFace repo. You should see an entry under `architectures` (e.g. `LlamaForCausalLM`).
+
+### Step 1: Clone the HuggingFace repository (optional)
+
+If the model is currently hosted in a HuggingFace repository, first clone that repository to download the raw model.
+
+```
+git lfs install
+git clone https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1
+cd Mistral-7B-Instruct-v0.1
+```
+
+### Step 2: Convert and quantize to a `.bin` file (optional, for PyTorch and Safetensors)
+
+If the model is in PyTorch or Safetensors format, a [Docker image](https://hub.docker.com/r/ollama/quantize) with the tooling required to convert and quantize models is available.
+
+First, Install [Docker](https://www.docker.com/get-started/).
+
+Next, to convert and quantize your model, run:
+
+```
+docker run --rm -v .:/model ollama/quantize -q q4_0 /model
+```
+
+This will output two files into the directory:
+
+- `f16.bin`: the model converted to GGUF
+- `q4_0.bin` the model quantized to a 4-bit quantization (we will use this file to create the Ollama model)
+
+### Step 3: Write a `Modelfile`
+
+Next, create a `Modelfile` for your model:
+
+```
+FROM ./q4_0.bin
+```
+
+(Optional) many chat models require a prompt template in order to answer correctly. A default prompt template can be specified with the `TEMPLATE` instruction in the `Modelfile`:
+
+```
+FROM ./q4_0.bin
+TEMPLATE "[INST] {{ .Prompt }} [/INST]"
+```
+
+### Step 4: Create the Ollama model
+
+Finally, create a model from your `Modelfile`:
+
+```
+ollama create example -f Modelfile
+```
+
+### Step 5: Run your model
+
+Next, test the model with `ollama run`:
+
+```
+ollama run example "What is your favourite condiment?"
+```
+
+## Publishing your model (optional – early alpha)
+
+Publishing models is in early alpha. If you'd like to publish your model to share with others, follow these steps:
+
+1. Create [an account](https://ollama.ai/signup)
+2. Run `cat ~/.ollama/id_ed25519.pub` to view your Ollama public key. Copy this to the clipboard.
+3. Add your public key to your [Ollama account](https://ollama.ai/settings/keys)
+
+Next, copy your model to your username's namespace:
+
+```
+ollama cp example <your username>/example
+```
+
+Then push the model:
+
+```
+ollama push <your username>/example
+```
+
+After publishing, your model will be available at `https://ollama.ai/<your username>/example`.
+
+## Quantization reference
+
+The quantization options are as follow (from highest highest to lowest levels of quantization). Note: some architectures such as Falcon do not support K quants.
+
+- `q2_K`
+- `q3_K`
+- `q3_K_S`
+- `q3_K_M`
+- `q3_K_L`
+- `q4_0` (recommended)
+- `q4_1`
+- `q4_K`
+- `q4_K_S`
+- `q4_K_M`
+- `q5_0`
+- `q5_1`
+- `q5_K`
+- `q5_K_S`
+- `q5_K_M`
+- `q6_K`
+- `q8_0`
+
+## Manually converting & quantizing models
+
+### Prerequisites
+
+Start by cloning the `llama.cpp` repo to your machine in another directory:
+
+```
+git clone https://github.com/ggerganov/llama.cpp.git
+cd llama.cpp
+```
+
+Next, install the Python dependencies:
+
+```
+pip install -r requirements.txt
+```
+
+Finally, build the `quantize` tool:
+
+```
+make quantize
+```
+
+### Convert the model
+
+Run the correct conversion script for your model architecture:
+
+```shell
+# LlamaForCausalLM or MistralForCausalLM
+python convert.py <path to model directory>
+
+# FalconForCausalLM
+python convert-falcon-hf-to-gguf.py <path to model directory>
+
+# GPTNeoXForCausalLM
+python convert-gptneox-hf-to-gguf.py <path to model directory>
+
+# GPTBigCodeForCausalLM
+python convert-starcoder-hf-to-gguf.py <path to model directory>
+```
+
+### Quantize the model
+
+```
+quantize <path to model dir>/ggml-model-f32.bin <path to model dir>/q4_0.bin q4_0
+```

docs/linux.md

@@ -0,0 +1,116 @@
+# Ollama on Linux
+
+## Install
+
+Install Ollama running this one-liner:
+>
+```bash
+curl https://ollama.ai/install.sh | sh
+```
+
+## Manual install
+
+### Download the `ollama` binary
+
+Ollama is distributed as a self-contained binary. Download it to a directory in your PATH:
+
+```bash
+sudo curl -L https://ollama.ai/download/ollama-linux-amd64 -o /usr/bin/ollama
+sudo chmod +x /usr/bin/ollama
+```
+
+### Adding Ollama as a startup service (recommended)
+
+Create a user for Ollama:
+
+```bash
+sudo useradd -r -s /bin/false -m -d /usr/share/ollama ollama
+```
+
+Create a service file in `/etc/systemd/system/ollama.service`:
+
+```ini
+[Unit]
+Description=Ollama Service
+After=network-online.target
+
+[Service]
+ExecStart=/usr/bin/ollama serve
+User=ollama
+Group=ollama
+Restart=always
+RestartSec=3
+
+[Install]
+WantedBy=default.target
+```
+
+Then start the service:
+
+```bash
+sudo systemctl daemon-reload
+sudo systemctl enable ollama
+```
+
+### Install CUDA drivers (optional – for Nvidia GPUs)
+
+[Download and install](https://developer.nvidia.com/cuda-downloads) CUDA.
+
+Verify that the drivers are installed by running the following command, which should print details about your GPU:
+
+```bash
+nvidia-smi
+```
+
+### Start Ollama
+
+Start Ollama using `systemd`:
+
+```bash
+sudo systemctl start ollama
+```
+
+## Update
+
+Update ollama by running the install script again:
+
+```bash
+curl https://ollama.ai/install.sh | sh
+```
+
+Or by downloading the ollama binary:
+
+```bash
+sudo curl -L https://ollama.ai/download/ollama-linux-amd64 -o /usr/bin/ollama
+sudo chmod +x /usr/bin/ollama
+```
+
+## Viewing logs
+
+To view logs of Ollama running as a startup service, run:
+
+```bash
+journalctl -u ollama
+```
+
+## Uninstall
+
+Remove the ollama service:
+
+```bash
+sudo systemctl stop ollama
+sudo systemctl disable ollama
+sudo rm /etc/systemd/system/ollama.service
+```
+
+Remove the ollama binary from your bin directory (either `/usr/local/bin`, `/usr/bin`, or `/bin`):
+
+```bash
+sudo rm $(which ollama)
+```
+
+Remove the downloaded models and Ollama service user:
+```bash
+sudo rm -r /usr/share/ollama
+sudo userdel ollama
+```

docs/modelfile.md

@@ -1,105 +1,179 @@
 # Ollama Model File
 
-> Note: this model file syntax is in development
+> Note: this `Modelfile` syntax is in development
 
 A model file is the blueprint to create and share models with Ollama.
 
+## Table of Contents
+
+- [Format](#format)
+- [Examples](#examples)
+- [Instructions](#instructions)
+  - [FROM (Required)](#from-required)
+    - [Build from llama2](#build-from-llama2)
+    - [Build from a bin file](#build-from-a-bin-file)
+  - [PARAMETER](#parameter)
+    - [Valid Parameters and Values](#valid-parameters-and-values)
+  - [TEMPLATE](#template)
+    - [Template Variables](#template-variables)
+  - [SYSTEM](#system)
+  - [ADAPTER](#adapter)
+  - [LICENSE](#license)
+- [Notes](#notes)
+
 ## Format
 
-The format of the Modelfile:
+The format of the `Modelfile`:
 
 ```modelfile
 # comment
 INSTRUCTION arguments
 ```
 
-| Instruction       | Description                                           |
-| ----------------- | ----------------------------------------------------- |
-| `FROM` (required) | Defines the base model to use                         |
-| `PARAMETER`       | Sets the parameters for how Ollama will run the model |
-| `SYSTEM`          | Specifies the system prompt that will set the context |
-| `TEMPLATE`        | The full prompt template to be sent to the model      |
-| `LICENSE`         | Specifies the legal license                           |
+| Instruction                         | Description                                                   |
+| ----------------------------------- | ------------------------------------------------------------- |
+| [`FROM`](#from-required) (required) | Defines the base model to use.                                |
+| [`PARAMETER`](#parameter)           | Sets the parameters for how Ollama will run the model.        |
+| [`TEMPLATE`](#template)             | The full prompt template to be sent to the model.             |
+| [`SYSTEM`](#system)                 | Specifies the system prompt that will be set in the template. |
+| [`ADAPTER`](#adapter)               | Defines the (Q)LoRA adapters to apply to the model.           |
+| [`LICENSE`](#license)               | Specifies the legal license.                                  |
 
 ## Examples
 
-An example of a model file creating a mario blueprint:
+An example of a `Modelfile` creating a mario blueprint:
 
-```
+```modelfile
 FROM llama2
 # sets the temperature to 1 [higher is more creative, lower is more coherent]
-# sets the context size to 4096
 PARAMETER temperature 1
+# sets the context window size to 4096, this controls how many tokens the LLM can use as context to generate the next token
 PARAMETER num_ctx 4096
 
-# Overriding the system prompt
+# sets a custom system prompt to specify the behavior of the chat assistant
 SYSTEM You are Mario from super mario bros, acting as an assistant.
 ```
 
 To use this:
 
-1. Save it as a file (eg. `Modelfile`)
-2. `ollama create NAME -f <location of the file eg. ./Modelfile>'`
-3. `ollama run NAME`
+1. Save it as a file (e.g. `Modelfile`)
+2. `ollama create choose-a-model-name -f <location of the file e.g. ./Modelfile>'`
+3. `ollama run choose-a-model-name`
 4. Start using the model!
 
-## FROM (Required)
+More examples are available in the [examples directory](../examples).
 
-The FROM instruction defines the base model to use when creating a model.
+## Instructions
 
-```
+### FROM (Required)
+
+The `FROM` instruction defines the base model to use when creating a model.
+
+```modelfile
 FROM <model name>:<tag>
 ```
 
-### Build from llama2
+#### Build from llama2
 
-```
+```modelfile
 FROM llama2
 ```
 
 A list of available base models:
 <https://github.com/jmorganca/ollama#model-library>
 
-### Build from a bin file
+#### Build from a `bin` file
 
-```
+```modelfile
 FROM ./ollama-model.bin
 ```
 
-## PARAMETER (Optional)
+This bin file location should be specified as an absolute path or relative to the `Modelfile` location.
+
+### PARAMETER
 
 The `PARAMETER` instruction defines a parameter that can be set when the model is run.
 
-```
+```modelfile
 PARAMETER <parameter> <parametervalue>
 ```
 
 ### Valid Parameters and Values
 
-| Parameter      | Description                                                                                                                                                                                                                                             | Value Type | Example Usage      |
-| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ------------------ |
-| num_ctx        | Sets the size of the prompt context size length model. (Default: 2048)                                                                                                                                                                                  | int        | num_ctx 4096       |
-| temperature    | The temperature of the model. Increasing the temperature will make the model answer more creatively. (Default: 0.8)                                                                                                                                     | float      | temperature 0.7    |
-| top_k          | Reduces the probability of generating nonsense. A higher value (e.g. 100) will give more diverse answers, while a lower value (e.g. 10) will be more conservative. (Default: 40)                                                                        | int        | top_k 40           |
-| top_p          | Works together with top-k. A higher value (e.g., 0.95) will lead to more diverse text, while a lower value (e.g., 0.5) will generate more focused and conservative text. (Default: 0.9)                                                                 | float      | top_p 0.9          |
-| num_gpu        | The number of GPUs to use. On macOS it defaults to 1 to enable metal support, 0 to disable.                                                                                                                                                             | int        | num_gpu 1          |
-| repeat_last_n  | Sets how far back for the model to look back to prevent repetition. (Default: 64, 0 = disabled, -1 = ctx-size)                                                                                                                                          | int        | repeat_last_n 64   |
-| repeat_penalty | Sets how strongly to penalize repetitions. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 0.9) will be more lenient. (Default: 1.1)                                                                     | float      | repeat_penalty 1.1 |
-| tfs_z          | Tail free sampling is used to reduce the impact of less probable tokens from the output. A higher value (e.g., 2.0) will reduce the impact more, while a value of 1.0 disables this setting. (default: 1)                                               | float      | tfs_z 1            |
-| mirostat       | Enable Mirostat sampling for controlling perplexity. (default: 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0)                                                                                                                                         | int        | mirostat 0         |
-| mirostat_tau   | Controls the balance between coherence and diversity of the output. A lower value will result in more focused and coherent text. (Default: 5.0)                                                                                                         | float      | mirostat_tau 5.0   |
-| mirostat_eta   | Influences how quickly the algorithm responds to feedback from the generated text. A lower learning rate will result in slower adjustments, while a higher learning rate will make the algorithm more responsive. (Default: 0.1)                        | float      | mirostat_eta 0.1   |
-| num_thread     | Sets the number of threads to use during computation. By default, Ollama will detect this for optimal performance. It is recommended to set this value to the number of physical CPU cores your system has (as opposed to the logical number of cores). | int        | num_thread 8       |
+| Parameter      | Description                                                                                                                                                                                                                                             | Value Type | Example Usage        |
+| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | -------------------- |
+| mirostat       | Enable Mirostat sampling for controlling perplexity. (default: 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0)                                                                                                                                         | int        | mirostat 0           |
+| mirostat_eta   | Influences how quickly the algorithm responds to feedback from the generated text. A lower learning rate will result in slower adjustments, while a higher learning rate will make the algorithm more responsive. (Default: 0.1)                        | float      | mirostat_eta 0.1     |
+| mirostat_tau   | Controls the balance between coherence and diversity of the output. A lower value will result in more focused and coherent text. (Default: 5.0)                                                                                                         | float      | mirostat_tau 5.0     |
+| num_ctx        | Sets the size of the context window used to generate the next token. (Default: 2048)                                                                                                                                                                    | int        | num_ctx 4096         |
+| num_gqa        | The number of GQA groups in the transformer layer. Required for some models, for example it is 8 for llama2:70b                                                                                                                                         | int        | num_gqa 1            |
+| num_gpu        | The number of layers to send to the GPU(s). On macOS it defaults to 1 to enable metal support, 0 to disable.                                                                                                                                            | int        | num_gpu 50           |
+| num_thread     | Sets the number of threads to use during computation. By default, Ollama will detect this for optimal performance. It is recommended to set this value to the number of physical CPU cores your system has (as opposed to the logical number of cores). | int        | num_thread 8         |
+| repeat_last_n  | Sets how far back for the model to look back to prevent repetition. (Default: 64, 0 = disabled, -1 = num_ctx)                                                                                                                                           | int        | repeat_last_n 64     |
+| repeat_penalty | Sets how strongly to penalize repetitions. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 0.9) will be more lenient. (Default: 1.1)                                                                     | float      | repeat_penalty 1.1   |
+| temperature    | The temperature of the model. Increasing the temperature will make the model answer more creatively. (Default: 0.8)                                                                                                                                     | float      | temperature 0.7      |
+| seed           | Sets the random number seed to use for generation. Setting this to a specific number will make the model generate the same text for the same prompt. (Default: 0)                                                                                       | int        | seed 42              |
+| stop           | Sets the stop sequences to use. When this pattern is encountered the LLM will stop generating text and return. Multiple stop patterns may be set by specifying multiple separate `stop` parameters in a modelfile.                                      | string     | stop "AI assistant:" |
+| tfs_z          | Tail free sampling is used to reduce the impact of less probable tokens from the output. A higher value (e.g., 2.0) will reduce the impact more, while a value of 1.0 disables this setting. (default: 1)                                               | float      | tfs_z 1              |
+| num_predict    | Maximum number of tokens to predict when generating text. (Default: 128, -1 = infinite generation, -2 = fill context)                                                                                                                                   | int        | num_predict 42       |
+| top_k          | Reduces the probability of generating nonsense. A higher value (e.g. 100) will give more diverse answers, while a lower value (e.g. 10) will be more conservative. (Default: 40)                                                                        | int        | top_k 40             |
+| top_p          | Works together with top-k. A higher value (e.g., 0.95) will lead to more diverse text, while a lower value (e.g., 0.5) will generate more focused and conservative text. (Default: 0.9)                                                                 | float      | top_p 0.9            |
 
-## Prompt
+### TEMPLATE
 
-When building on top of the base models supplied by Ollama, it comes with the prompt template predefined. To override the supplied system prompt, simply add `SYSTEM insert system prompt` to change the system prompt.
+`TEMPLATE` of the full prompt template to be passed into the model. It may include (optionally) a system prompt and a user's prompt. This is used to create a full custom prompt, and syntax may be model specific. You can usually find the template for a given model in the readme for that model.
 
-### Prompt Template
+#### Template Variables
 
-`TEMPLATE` the full prompt template to be passed into the model. It may include (optionally) a system prompt, user prompt, and assistant prompt. This is used to create a full custom prompt, and syntax may be model specific.
+| Variable        | Description                                                                                                  |
+| --------------- | ------------------------------------------------------------------------------------------------------------ |
+| `{{ .System }}` | The system prompt used to specify custom behavior, this must also be set in the Modelfile as an instruction. |
+| `{{ .Prompt }}` | The incoming prompt, this is not specified in the model file and will be set based on input.                 |
+| `{{ .First }}`  | A boolean value used to render specific template information for the first generation of a session.          |
+
+```modelfile
+TEMPLATE """
+{{- if .First }}
+### System:
+{{ .System }}
+{{- end }}
+
+### User:
+{{ .Prompt }}
+
+### Response:
+"""
+
+SYSTEM """<system message>"""
+```
+
+### SYSTEM
+
+The `SYSTEM` instruction specifies the system prompt to be used in the template, if applicable.
+
+```modelfile
+SYSTEM """<system message>"""
+```
+
+### ADAPTER
+
+The `ADAPTER` instruction specifies the LoRA adapter to apply to the base model. The value of this instruction should be an absolute path or a path relative to the Modelfile and the file must be in a GGML file format. The adapter should be tuned from the base model otherwise the behaviour is undefined.
+
+```modelfile
+ADAPTER ./ollama-lora.bin
+```
+
+### LICENSE
+
+The `LICENSE` instruction allows you to specify the legal license under which the model used with this Modelfile is shared or distributed.
+
+```modelfile
+LICENSE """
+<license text>
+"""
+```
 
 ## Notes
 
-- the **modelfile is not case sensitive**. In the examples, we use uppercase for instructions to make it easier to distinguish it from arguments.
+- the **`Modelfile` is not case sensitive**. In the examples, we use uppercase for instructions to make it easier to distinguish it from arguments.
 - Instructions can be in any order. In the examples, we start with FROM instruction to keep it easily readable.

docs/tutorials.md

@@ -0,0 +1,8 @@
+# Tutorials
+
+Here is a list of ways you can use Ollama with other tools to build interesting applications.
+
+- [Using LangChain with Ollama in JavaScript](./tutorials/langchainjs.md)
+- [Using LangChain with Ollama in Python](./tutorials/langchainpy.md)
+
+Also be sure to check out the [examples](../examples) directory for more ways to use Ollama.

docs/tutorials/langchainjs.md

@@ -0,0 +1,77 @@
+# Using LangChain with Ollama using JavaScript
+
+In this tutorial, we are going to use JavaScript with LangChain and Ollama to learn about something just a touch more recent. In August 2023, there was a series of wildfires on Maui. There is no way an LLM trained before that time can know about this, since their training data would not include anything as recent as that. So we can find the [Wikipedia article about the fires](https://en.wikipedia.org/wiki/2023_Hawaii_wildfires) and ask questions about the contents.
+
+To get started, let's just use **LangChain** to ask a simple question to a model. To do this with JavaScript, we need to install **LangChain**:
+
+```bash
+npm install langchain
+```
+
+Now we can start building out our JavaScript:
+
+```javascript
+import { Ollama } from "langchain/llms/ollama";
+
+const ollama = new Ollama({
+  baseUrl: "http://localhost:11434",
+  model: "llama2",
+});
+
+const answer = await ollama.call(`why is the sky blue?`);
+
+console.log(answer);
+```
+
+That will get us the same thing as if we ran `ollama run llama2 "why is the sky blue"` in the terminal. But we want to load a document from the web to ask a question against. **Cheerio** is a great library for ingesting a webpage, and **LangChain** uses it in their **CheerioWebBaseLoader**. So let's install **Cheerio** and build that part of the app.
+
+```bash
+npm install cheerio 
+```
+
+```javascript
+import { CheerioWebBaseLoader } from "langchain/document_loaders/web/cheerio";
+
+const loader = new CheerioWebBaseLoader("https://en.wikipedia.org/wiki/2023_Hawaii_wildfires");
+const data = await loader.load();
+```
+
+That will load the document. Although this page is smaller than the Odyssey, it is certainly bigger than the context size for most LLMs. So we are going to need to split into smaller pieces, and then select just the pieces relevant to our question. This is a great use for a vector datastore. In this example, we will use the **MemoryVectorStore** that is part of **LangChain**. But there is one more thing we need to get the content into the datastore. We have to run an embeddings process that converts the tokens in the text into a series of vectors. And for that, we are going to use **Tensorflow**. There is a lot of stuff going on in this one. First, install the **Tensorflow** components that we need.
+
+```javascript
+npm install @tensorflow/tfjs-core@3.6.0 @tensorflow/tfjs-converter@3.6.0 @tensorflow-models/universal-sentence-encoder@1.3.3 @tensorflow/tfjs-node@4.10.0
+```
+
+If you just install those components without the version numbers, it will install the latest versions, but there are conflicts within **Tensorflow**, so you need to install the compatible versions.
+
+```javascript
+import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"
+import { MemoryVectorStore } from "langchain/vectorstores/memory";
+import "@tensorflow/tfjs-node";
+import { TensorFlowEmbeddings } from "langchain/embeddings/tensorflow";
+
+// Split the text into 500 character chunks. And overlap each chunk by 20 characters
+const textSplitter = new RecursiveCharacterTextSplitter({
+ chunkSize: 500,
+ chunkOverlap: 20
+});
+const splitDocs = await textSplitter.splitDocuments(data);
+
+// Then use the TensorFlow Embedding to store these chunks in the datastore
+const vectorStore = await MemoryVectorStore.fromDocuments(splitDocs, new TensorFlowEmbeddings());
+```
+
+To connect the datastore to a question asked to a LLM, we need to use the concept at the heart of **LangChain**: the chain. Chains are a way to connect a number of activities together to accomplish a particular tasks. There are a number of chain types available, but for this tutorial we are using the **RetrievalQAChain**.
+
+```javascript
+import { RetrievalQAChain } from "langchain/chains";
+
+const retriever = vectorStore.asRetriever();
+const chain = RetrievalQAChain.fromLLM(ollama, retriever);
+const result = await chain.call({query: "When was Hawaii's request for a major disaster declaration approved?"});
+console.log(result.text)
+```
+
+So we created a retriever, which is a way to return the chunks that match a query from a datastore. And then connect the retriever and the model via a chain. Finally, we send a query to the chain, which results in an answer using our document as a source. The answer it returned was correct, August 10, 2023.
+
+And that is a simple introduction to what you can do with **LangChain** and **Ollama.**

docs/tutorials/langchainpy.md

@@ -0,0 +1,81 @@
+# Using LangChain with Ollama in Python
+
+Let's imagine we are studying the classics, such as **the Odyssey** by **Homer**. We might have a question about Neleus and his family. If you ask llama2 for that info, you may get something like:
+
+> I apologize, but I'm a large language model, I cannot provide information on individuals or families that do not exist in reality. Neleus is not a real person or character, and therefore does not have a family or any other personal details. My apologies for any confusion. Is there anything else I can help you with?
+
+This sounds like a typical censored response, but even llama2-uncensored gives a mediocre answer:
+
+> Neleus was a legendary king of Pylos and the father of Nestor, one of the Argonauts. His mother was Clymene, a sea nymph, while his father was Neptune, the god of the sea.
+
+So let's figure out how we can use **LangChain** with Ollama to ask our question to the actual document, the Odyssey by Homer, using Python.
+
+Let's start by asking a simple question that we can get an answer to from the **Llama2** model using **Ollama**. First, we need to install the **LangChain** package:
+
+`pip install langchain`
+
+Then we can create a model and ask the question:
+
+```python
+from langchain.llms import Ollama
+ollama = Ollama(base_url='http://localhost:11434',
+model="llama2")
+print(ollama("why is the sky blue"))
+```
+
+Notice that we are defining the model and the base URL for Ollama.
+
+Now let's load a document to ask questions against. I'll load up the Odyssey by Homer, which you can find at Project Gutenberg. We will need **WebBaseLoader** which is part of **LangChain** and loads text from any webpage. On my machine, I also needed to install **bs4** to get that to work, so run `pip install bs4`.
+
+```python
+from langchain.document_loaders import WebBaseLoader
+loader = WebBaseLoader("https://www.gutenberg.org/files/1727/1727-h/1727-h.htm")
+data = loader.load()
+```
+
+This file is pretty big. Just the preface is 3000 tokens. Which means the full document won't fit into the context for the model. So we need to split it up into smaller pieces.
+
+```python
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+text_splitter=RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=0)
+all_splits = text_splitter.split_documents(data)
+```
+
+It's split up, but we have to find the relevant splits and then submit those to the model. We can do this by creating embeddings and storing them in a vector database. For now, we don't have embeddings built in to Ollama, though we will be adding that soon, so for now, we can use the GPT4All library for that. We will use ChromaDB in this example for a vector database. `pip install GPT4All chromadb`
+
+```python
+from langchain.embeddings import GPT4AllEmbeddings
+from langchain.vectorstores import Chroma
+vectorstore = Chroma.from_documents(documents=all_splits, embedding=GPT4AllEmbeddings())
+```
+
+Now let's ask a question from the document. **Who was Neleus, and who is in his family?** Neleus is a character in the Odyssey, and the answer can be found in our text.
+
+```python
+question="Who is Neleus and who is in Neleus' family?"
+docs = vectorstore.similarity_search(question)
+len(docs)
+```
+
+This will output the number of matches for chunks of data similar to the search.
+
+The next thing is to send the question and the relevant parts of the docs to the model to see if we can get a good answer. But we are stitching two parts of the process together, and that is called a chain. This means we need to define a chain:
+
+```python
+from langchain.chains import RetrievalQA
+qachain=RetrievalQA.from_chain_type(ollama, retriever=vectorstore.as_retriever())
+qachain({"query": question})
+```
+
+The answer received from this chain was:
+
+> Neleus is a character in Homer's "Odyssey" and is mentioned in the context of Penelope's suitors. Neleus is the father of Chloris, who is married to Neleus and bears him several children, including Nestor, Chromius, Periclymenus, and Pero. Amphinomus, the son of Nisus, is also mentioned as a suitor of Penelope and is known for his good natural disposition and agreeable conversation.
+
+It's not a perfect answer, as it implies Neleus married his daughter when actually Chloris "was the youngest daughter to Amphion son of Iasus and king of Minyan Orchomenus, and was Queen in Pylos".
+
+I updated the chunk_overlap for the text splitter to 20 and tried again and got a much better answer:
+
+> Neleus is a character in Homer's epic poem "The Odyssey." He is the husband of Chloris, who is the youngest daughter of Amphion son of Iasus and king of Minyan Orchomenus. Neleus has several children with Chloris, including Nestor, Chromius, Periclymenus, and Pero.
+
+And that is a much better answer.

examples/.gitignore

@@ -0,0 +1,171 @@
+node_modules
+# OSX
+.DS_STORE
+
+# Models
+models/
+
+# Local Chroma db
+.chroma/
+db/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

examples/README.md

@@ -1,15 +1,3 @@
 # Examples
 
-This directory contains examples that can be created and run with `ollama`.
-
-To create a model:
-
-```
-ollama create example -f <example file>
-```
-
-To run a model:
-
-```
-ollama run example
-```
+This directory contains different examples of using Ollama.

examples/golang-simplegenerate/main.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+)
+
+func main() {
+	body := []byte(`{"model":"mistral"}`)
+	resp, err := http.Post("http://localhost:11434/api/generate", "application/json", bytes.NewBuffer(body))
+
+	if err != nil {
+		fmt.Print(err.Error())
+		os.Exit(1)
+	}
+
+	responseData, err := io.ReadAll(resp.Body)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(string(responseData))
+
+}

examples/kubernetes/README.md

@@ -0,0 +1,36 @@
+# Deploy Ollama to Kubernetes
+
+## Prerequisites
+
+- Ollama: https://ollama.ai/download
+- Kubernetes cluster. This example will use Google Kubernetes Engine.
+
+## Steps
+
+1. Create the Ollama namespace, daemon set, and service
+
+    ```bash
+    kubectl apply -f cpu.yaml
+    ```
+
+1. Port forward the Ollama service to connect and use it locally
+
+    ```bash
+    kubectl -n ollama port-forward service/ollama 11434:80
+    ```
+
+1. Pull and run a model, for example `orca-mini:3b`
+
+    ```bash
+    ollama run orca-mini:3b
+    ```
+
+## (Optional) Hardware Acceleration
+
+Hardware acceleration in Kubernetes requires NVIDIA's [`k8s-device-plugin`](https://github.com/NVIDIA/k8s-device-plugin). Follow the link for more details.
+
+Once configured, create a GPU enabled Ollama deployment.
+
+```bash
+kubectl apply -f gpu.yaml
+```

examples/kubernetes/cpu.yaml

@@ -0,0 +1,42 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ollama
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ollama
+  namespace: ollama
+spec:
+  selector:
+    matchLabels:
+      name: ollama
+  template:
+    metadata:
+      labels:
+        name: ollama
+    spec:
+      containers:
+      - name: ollama
+        image: ollama/ollama:latest
+        ports:
+        - name: http
+          containerPort: 11434
+          protocol: TCP
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ollama
+  namespace: ollama
+spec:
+  type: ClusterIP
+  selector:
+    name: ollama
+  ports:
+  - port: 80
+    name: http
+    targetPort: http
+    protocol: TCP

examples/kubernetes/gpu.yaml

@@ -0,0 +1,56 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ollama
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ollama
+  namespace: ollama
+spec:
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      name: ollama
+  template:
+    metadata:
+      labels:
+        name: ollama
+    spec:
+      containers:
+      - name: ollama
+        image: ollama/ollama:latest
+        env:
+        - name: PATH
+          value: /usr/local/nvidia/bin:/usr/local/nvidia/lib64:/usr/bin:/usr/sbin:/bin:/sbin
+        - name: LD_LIBRARY_PATH
+          value: /usr/local/nvidia/lib64
+        ports:
+        - name: http
+          containerPort: 11434
+          protocol: TCP
+        resources:
+          limits:
+            nvidia.com/gpu: 1
+      tolerations:
+      - key: nvidia.com/gpu
+        operator: Exists
+        effect: NoSchedule
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ollama
+  namespace: ollama
+spec:
+  type: ClusterIP
+  selector:
+    name: ollama
+  ports:
+  - port: 80
+    name: http
+    targetPort: http
+    protocol: TCP

examples/langchain-python-rag-document/README.md

@@ -0,0 +1,21 @@
+# LangChain Document QA
+
+This example provides an interface for asking questions to a PDF document.
+
+## Setup
+
+```
+pip install -r requirements.txt
+```
+
+## Run
+
+```
+python main.py
+```
+
+A prompt will appear, where questions may be asked:
+
+```
+Query: How many locations does WeWork have?
+```

examples/langchain-python-rag-document/main.py

@@ -0,0 +1,61 @@
+from langchain.document_loaders import OnlinePDFLoader
+from langchain.vectorstores import Chroma
+from langchain.embeddings import GPT4AllEmbeddings
+from langchain import PromptTemplate
+from langchain.llms import Ollama
+from langchain.callbacks.manager import CallbackManager
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+from langchain.chains import RetrievalQA
+import sys
+import os
+
+class SuppressStdout:
+    def __enter__(self):
+        self._original_stdout = sys.stdout
+        self._original_stderr = sys.stderr
+        sys.stdout = open(os.devnull, 'w')
+        sys.stderr = open(os.devnull, 'w')
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        sys.stdout.close()
+        sys.stdout = self._original_stdout
+        sys.stderr = self._original_stderr
+
+# load the pdf and split it into chunks
+loader = OnlinePDFLoader("https://d18rn0p25nwr6d.cloudfront.net/CIK-0001813756/975b3e9b-268e-4798-a9e4-2a9a7c92dc10.pdf")
+data = loader.load()
+
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=0)
+all_splits = text_splitter.split_documents(data)
+
+with SuppressStdout():
+    vectorstore = Chroma.from_documents(documents=all_splits, embedding=GPT4AllEmbeddings())
+
+while True:
+    query = input("\nQuery: ")
+    if query == "exit":
+        break
+    if query.strip() == "":
+        continue
+
+    # Prompt
+    template = """Use the following pieces of context to answer the question at the end. 
+    If you don't know the answer, just say that you don't know, don't try to make up an answer. 
+    Use three sentences maximum and keep the answer as concise as possible. 
+    {context}
+    Question: {question}
+    Helpful Answer:"""
+    QA_CHAIN_PROMPT = PromptTemplate(
+        input_variables=["context", "question"],
+        template=template,
+    )
+
+    llm = Ollama(model="llama2:13b", callback_manager=CallbackManager([StreamingStdOutCallbackHandler()]))
+    qa_chain = RetrievalQA.from_chain_type(
+        llm,
+        retriever=vectorstore.as_retriever(),
+        chain_type_kwargs={"prompt": QA_CHAIN_PROMPT},
+    )
+
+    result = qa_chain({"query": query})

examples/langchain-python-rag-document/requirements.txt

@@ -0,0 +1,109 @@
+absl-py==1.4.0
+aiohttp==3.8.5
+aiosignal==1.3.1
+anyio==3.7.1
+astunparse==1.6.3
+async-timeout==4.0.3
+attrs==23.1.0
+backoff==2.2.1
+beautifulsoup4==4.12.2
+bs4==0.0.1
+cachetools==5.3.1
+certifi==2023.7.22
+cffi==1.15.1
+chardet==5.2.0
+charset-normalizer==3.2.0
+Chroma==0.2.0
+chroma-hnswlib==0.7.2
+chromadb==0.4.5
+click==8.1.6
+coloredlogs==15.0.1
+cryptography==41.0.3
+dataclasses-json==0.5.14
+fastapi==0.99.1
+filetype==1.2.0
+flatbuffers==23.5.26
+frozenlist==1.4.0
+gast==0.4.0
+google-auth==2.22.0
+google-auth-oauthlib==1.0.0
+google-pasta==0.2.0
+gpt4all==1.0.8
+grpcio==1.57.0
+h11==0.14.0
+h5py==3.9.0
+httptools==0.6.0
+humanfriendly==10.0
+idna==3.4
+importlib-resources==6.0.1
+joblib==1.3.2
+keras==2.13.1
+langchain==0.0.261
+langsmith==0.0.21
+libclang==16.0.6
+lxml==4.9.3
+Markdown==3.4.4
+MarkupSafe==2.1.3
+marshmallow==3.20.1
+monotonic==1.6
+mpmath==1.3.0
+multidict==6.0.4
+mypy-extensions==1.0.0
+nltk==3.8.1
+numexpr==2.8.5
+numpy==1.24.3
+oauthlib==3.2.2
+onnxruntime==1.15.1
+openapi-schema-pydantic==1.2.4
+opt-einsum==3.3.0
+overrides==7.4.0
+packaging==23.1
+pdf2image==1.16.3
+pdfminer==20191125
+pdfminer.six==20221105
+Pillow==10.0.0
+posthog==3.0.1
+protobuf==4.24.0
+pulsar-client==3.2.0
+pyasn1==0.5.0
+pyasn1-modules==0.3.0
+pycparser==2.21
+pycryptodome==3.18.0
+pydantic==1.10.12
+PyPika==0.48.9
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-magic==0.4.27
+PyYAML==6.0.1
+regex==2023.8.8
+requests==2.31.0
+requests-oauthlib==1.3.1
+rsa==4.9
+six==1.16.0
+sniffio==1.3.0
+soupsieve==2.4.1
+SQLAlchemy==2.0.19
+starlette==0.27.0
+sympy==1.12
+tabulate==0.9.0
+tenacity==8.2.2
+tensorboard==2.13.0
+tensorboard-data-server==0.7.1
+tensorflow==2.13.0
+tensorflow-estimator==2.13.0
+tensorflow-hub==0.14.0
+tensorflow-macos==2.13.0
+termcolor==2.3.0
+tokenizers==0.13.3
+tqdm==4.66.1
+typing-inspect==0.9.0
+typing_extensions==4.5.0
+unstructured==0.9.2
+urllib3==1.26.16
+uvicorn==0.23.2
+uvloop==0.17.0
+watchfiles==0.19.0
+websockets==11.0.3
+Werkzeug==2.3.6
+wrapt==1.15.0
+yarl==1.9.2

examples/langchain-python-rag-privategpt/.gitignore

@@ -0,0 +1,170 @@
+# OSX
+.DS_STORE
+
+# Models
+models/
+
+# Local Chroma db
+.chroma/
+db/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

examples/langchain-python-rag-privategpt/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

examples/langchain-python-rag-privategpt/README.md

@@ -0,0 +1,91 @@
+# PrivateGPT with Llama 2 uncensored
+
+https://github.com/jmorganca/ollama/assets/3325447/20cf8ec6-ff25-42c6-bdd8-9be594e3ce1b
+
+> Note: this example is a slightly modified version of PrivateGPT using models such as Llama 2 Uncensored. All credit for PrivateGPT goes to Iván Martínez who is the creator of it, and you can find his GitHub repo [here](https://github.com/imartinez/privateGPT).
+
+### Setup
+
+Set up a virtual environment (optional):
+
+```
+python3 -m venv .venv
+source .venv/bin/activate
+```
+
+Install the Python dependencies:
+
+```shell
+pip install -r requirements.txt
+```
+
+Pull the model you'd like to use:
+
+```
+ollama pull llama2-uncensored
+```
+
+### Getting WeWork's latest quarterly earnings report (10-Q)
+
+```
+mkdir source_documents
+curl https://d18rn0p25nwr6d.cloudfront.net/CIK-0001813756/975b3e9b-268e-4798-a9e4-2a9a7c92dc10.pdf -o source_documents/wework.pdf
+```
+
+### Ingesting files
+
+```shell
+python ingest.py
+```
+
+Output should look like this:
+
+```shell
+Creating new vectorstore
+Loading documents from source_documents
+Loading new documents: 100%|██████████████████████| 1/1 [00:01<00:00,  1.73s/it]
+Loaded 1 new documents from source_documents
+Split into 90 chunks of text (max. 500 tokens each)
+Creating embeddings. May take some minutes...
+Using embedded DuckDB with persistence: data will be stored in: db
+Ingestion complete! You can now run privateGPT.py to query your documents
+```
+
+### Ask questions
+
+```shell
+python privateGPT.py
+
+Enter a query: How many locations does WeWork have?
+
+> Answer (took 17.7 s.):
+As of June 2023, WeWork has 777 locations worldwide, including 610 Consolidated Locations (as defined in the section entitled Key Performance Indicators).
+```
+
+### Try a different model:
+
+```
+ollama pull llama2:13b
+MODEL=llama2:13b python privateGPT.py
+```
+
+## Adding more files
+
+Put any and all your files into the `source_documents` directory
+
+The supported extensions are:
+
+- `.csv`: CSV,
+- `.docx`: Word Document,
+- `.doc`: Word Document,
+- `.enex`: EverNote,
+- `.eml`: Email,
+- `.epub`: EPub,
+- `.html`: HTML File,
+- `.md`: Markdown,
+- `.msg`: Outlook Message,
+- `.odt`: Open Document Text,
+- `.pdf`: Portable Document Format (PDF),
+- `.pptx` : PowerPoint Document,
+- `.ppt` : PowerPoint Document,
+- `.txt`: Text file (UTF-8),

examples/langchain-python-rag-privategpt/constants.py

@@ -0,0 +1,11 @@
+import os
+from chromadb.config import Settings
+
+# Define the folder for storing database
+PERSIST_DIRECTORY = os.environ.get('PERSIST_DIRECTORY', 'db')
+
+# Define the Chroma settings
+CHROMA_SETTINGS = Settings(
+        persist_directory=PERSIST_DIRECTORY,
+        anonymized_telemetry=False
+)

examples/langchain-python-rag-privategpt/ingest.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+import os
+import glob
+from typing import List
+from multiprocessing import Pool
+from tqdm import tqdm
+
+from langchain.document_loaders import (
+    CSVLoader,
+    EverNoteLoader,
+    PyMuPDFLoader,
+    TextLoader,
+    UnstructuredEmailLoader,
+    UnstructuredEPubLoader,
+    UnstructuredHTMLLoader,
+    UnstructuredMarkdownLoader,
+    UnstructuredODTLoader,
+    UnstructuredPowerPointLoader,
+    UnstructuredWordDocumentLoader,
+)
+
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.vectorstores import Chroma
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.docstore.document import Document
+from constants import CHROMA_SETTINGS
+
+
+# Load environment variables
+persist_directory = os.environ.get('PERSIST_DIRECTORY', 'db')
+source_directory = os.environ.get('SOURCE_DIRECTORY', 'source_documents')
+embeddings_model_name = os.environ.get('EMBEDDINGS_MODEL_NAME', 'all-MiniLM-L6-v2')
+chunk_size = 500
+chunk_overlap = 50
+
+# Custom document loaders
+class MyElmLoader(UnstructuredEmailLoader):
+    """Wrapper to fallback to text/plain when default does not work"""
+
+    def load(self) -> List[Document]:
+        """Wrapper adding fallback for elm without html"""
+        try:
+            try:
+                doc = UnstructuredEmailLoader.load(self)
+            except ValueError as e:
+                if 'text/html content not found in email' in str(e):
+                    # Try plain text
+                    self.unstructured_kwargs["content_source"]="text/plain"
+                    doc = UnstructuredEmailLoader.load(self)
+                else:
+                    raise
+        except Exception as e:
+            # Add file_path to exception message
+            raise type(e)(f"{self.file_path}: {e}") from e
+
+        return doc
+
+
+# Map file extensions to document loaders and their arguments
+LOADER_MAPPING = {
+    ".csv": (CSVLoader, {}),
+    # ".docx": (Docx2txtLoader, {}),
+    ".doc": (UnstructuredWordDocumentLoader, {}),
+    ".docx": (UnstructuredWordDocumentLoader, {}),
+    ".enex": (EverNoteLoader, {}),
+    ".eml": (MyElmLoader, {}),
+    ".epub": (UnstructuredEPubLoader, {}),
+    ".html": (UnstructuredHTMLLoader, {}),
+    ".md": (UnstructuredMarkdownLoader, {}),
+    ".odt": (UnstructuredODTLoader, {}),
+    ".pdf": (PyMuPDFLoader, {}),
+    ".ppt": (UnstructuredPowerPointLoader, {}),
+    ".pptx": (UnstructuredPowerPointLoader, {}),
+    ".txt": (TextLoader, {"encoding": "utf8"}),
+    # Add more mappings for other file extensions and loaders as needed
+}
+
+
+def load_single_document(file_path: str) -> List[Document]:
+    ext = "." + file_path.rsplit(".", 1)[-1]
+    if ext in LOADER_MAPPING:
+        loader_class, loader_args = LOADER_MAPPING[ext]
+        loader = loader_class(file_path, **loader_args)
+        return loader.load()
+
+    raise ValueError(f"Unsupported file extension '{ext}'")
+
+def load_documents(source_dir: str, ignored_files: List[str] = []) -> List[Document]:
+    """
+    Loads all documents from the source documents directory, ignoring specified files
+    """
+    all_files = []
+    for ext in LOADER_MAPPING:
+        all_files.extend(
+            glob.glob(os.path.join(source_dir, f"**/*{ext}"), recursive=True)
+        )
+    filtered_files = [file_path for file_path in all_files if file_path not in ignored_files]
+
+    with Pool(processes=os.cpu_count()) as pool:
+        results = []
+        with tqdm(total=len(filtered_files), desc='Loading new documents', ncols=80) as pbar:
+            for i, docs in enumerate(pool.imap_unordered(load_single_document, filtered_files)):
+                results.extend(docs)
+                pbar.update()
+
+    return results
+
+def process_documents(ignored_files: List[str] = []) -> List[Document]:
+    """
+    Load documents and split in chunks
+    """
+    print(f"Loading documents from {source_directory}")
+    documents = load_documents(source_directory, ignored_files)
+    if not documents:
+        print("No new documents to load")
+        exit(0)
+    print(f"Loaded {len(documents)} new documents from {source_directory}")
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+    texts = text_splitter.split_documents(documents)
+    print(f"Split into {len(texts)} chunks of text (max. {chunk_size} tokens each)")
+    return texts
+
+def does_vectorstore_exist(persist_directory: str) -> bool:
+    """
+    Checks if vectorstore exists
+    """
+    if os.path.exists(os.path.join(persist_directory, 'index')):
+        if os.path.exists(os.path.join(persist_directory, 'chroma-collections.parquet')) and os.path.exists(os.path.join(persist_directory, 'chroma-embeddings.parquet')):
+            list_index_files = glob.glob(os.path.join(persist_directory, 'index/*.bin'))
+            list_index_files += glob.glob(os.path.join(persist_directory, 'index/*.pkl'))
+            # At least 3 documents are needed in a working vectorstore
+            if len(list_index_files) > 3:
+                return True
+    return False
+
+def main():
+    # Create embeddings
+    embeddings = HuggingFaceEmbeddings(model_name=embeddings_model_name)
+
+    if does_vectorstore_exist(persist_directory):
+        # Update and store locally vectorstore
+        print(f"Appending to existing vectorstore at {persist_directory}")
+        db = Chroma(persist_directory=persist_directory, embedding_function=embeddings, client_settings=CHROMA_SETTINGS)
+        collection = db.get()
+        texts = process_documents([metadata['source'] for metadata in collection['metadatas']])
+        print(f"Creating embeddings. May take some minutes...")
+        db.add_documents(texts)
+    else:
+        # Create and store locally vectorstore
+        print("Creating new vectorstore")
+        texts = process_documents()
+        print(f"Creating embeddings. May take some minutes...")
+        db = Chroma.from_documents(texts, embeddings, persist_directory=persist_directory)
+    db.persist()
+    db = None
+
+    print(f"Ingestion complete! You can now run privateGPT.py to query your documents")
+
+
+if __name__ == "__main__":
+    main()

examples/langchain-python-rag-privategpt/privateGPT.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+from langchain.chains import RetrievalQA
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+from langchain.vectorstores import Chroma
+from langchain.llms import Ollama
+import chromadb
+import os
+import argparse
+import time
+
+model = os.environ.get("MODEL", "llama2-uncensored")
+# For embeddings model, the example uses a sentence-transformers model
+# https://www.sbert.net/docs/pretrained_models.html 
+# "The all-mpnet-base-v2 model provides the best quality, while all-MiniLM-L6-v2 is 5 times faster and still offers good quality."
+embeddings_model_name = os.environ.get("EMBEDDINGS_MODEL_NAME", "all-MiniLM-L6-v2")
+persist_directory = os.environ.get("PERSIST_DIRECTORY", "db")
+target_source_chunks = int(os.environ.get('TARGET_SOURCE_CHUNKS',4))
+
+from constants import CHROMA_SETTINGS
+
+def main():
+    # Parse the command line arguments
+    args = parse_arguments()
+    embeddings = HuggingFaceEmbeddings(model_name=embeddings_model_name)
+
+    db = Chroma(persist_directory=persist_directory, embedding_function=embeddings)
+
+    retriever = db.as_retriever(search_kwargs={"k": target_source_chunks})
+    # activate/deactivate the streaming StdOut callback for LLMs
+    callbacks = [] if args.mute_stream else [StreamingStdOutCallbackHandler()]
+
+    llm = Ollama(model=model, callbacks=callbacks)
+
+    qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=retriever, return_source_documents= not args.hide_source)
+    # Interactive questions and answers
+    while True:
+        query = input("\nEnter a query: ")
+        if query == "exit":
+            break
+        if query.strip() == "":
+            continue
+
+        # Get the answer from the chain
+        start = time.time()
+        res = qa(query)
+        answer, docs = res['result'], [] if args.hide_source else res['source_documents']
+        end = time.time()
+
+        # Print the result
+        print("\n\n> Question:")
+        print(query)
+        print(answer)
+
+        # Print the relevant sources used for the answer
+        for document in docs:
+            print("\n> " + document.metadata["source"] + ":")
+            print(document.page_content)
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description='privateGPT: Ask questions to your documents without an internet connection, '
+                                                 'using the power of LLMs.')
+    parser.add_argument("--hide-source", "-S", action='store_true',
+                        help='Use this flag to disable printing of source documents used for answers.')
+
+    parser.add_argument("--mute-stream", "-M",
+                        action='store_true',
+                        help='Use this flag to disable the streaming StdOut callback for LLMs.')
+
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    main()

examples/langchain-python-rag-privategpt/pyproject.toml

@@ -0,0 +1,26 @@
+[tool.poetry]
+name = "privategpt"
+version = "0.1.0"
+description = ""
+authors = ["Ivan Martinez <ivanmartit@gmail.com>"]
+license = "Apache Version 2.0"
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+langchain = "0.0.261"
+gpt4all = "^1.0.3"
+chromadb = "^0.3.26"
+PyMuPDF = "^1.22.5"
+python-dotenv = "^1.0.0"
+unstructured = "^0.8.0"
+extract-msg = "^0.41.5"
+tabulate = "^0.9.0"
+pandoc = "^2.3"
+pypandoc = "^1.11"
+tqdm = "^4.65.0"
+sentence-transformers = "^2.2.2"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

examples/langchain-python-rag-privategpt/requirements.txt

@@ -0,0 +1,14 @@
+langchain==0.0.274
+gpt4all==1.0.8
+chromadb==0.4.7
+llama-cpp-python==0.1.81
+urllib3==2.0.4
+PyMuPDF==1.23.5
+python-dotenv==1.0.0
+unstructured==0.10.8
+extract-msg==0.45.0
+tabulate==0.9.0
+pandoc==2.3
+pypandoc==1.11
+tqdm==4.66.1
+sentence_transformers==2.2.2

examples/langchain-python-rag-websummary/README.md

@@ -0,0 +1,15 @@
+# LangChain Web Summarization
+
+This example summarizes a website
+
+## Setup
+
+```
+pip install -r requirements.txt
+```
+
+## Run
+
+```
+python main.py
+```

examples/langchain-python-rag-websummary/main.py

@@ -0,0 +1,12 @@
+from langchain.llms import Ollama
+from langchain.document_loaders import WebBaseLoader
+from langchain.chains.summarize import load_summarize_chain
+
+loader = WebBaseLoader("https://ollama.ai/blog/run-llama2-uncensored-locally")
+docs = loader.load()
+
+llm = Ollama(model="llama2")
+chain = load_summarize_chain(llm, chain_type="stuff")
+
+result = chain.run(docs)
+print(result)

examples/langchain-python-rag-websummary/requirements.txt

@@ -0,0 +1,2 @@
+langchain==0.0.259
+bs4==0.0.1

examples/langchain-python-simple/README.md

@@ -0,0 +1,21 @@
+# LangChain
+
+This example is a basic "hello world" of using LangChain with Ollama.
+
+## Setup
+
+```
+pip install -r requirements.txt
+```
+
+## Run
+
+```
+python main.py
+```
+
+Running this example will print the response for "hello":
+
+```
+Hello! It's nice to meet you. hopefully you are having a great day! Is there something I can help you with or would you like to chat?
+```

examples/langchain-python-simple/main.py

@@ -0,0 +1,4 @@
+from langchain.llms import Ollama
+llm = Ollama(model="llama2")
+res = llm.predict("hello")
+print (res)

examples/langchain-python-simple/requirements.txt

@@ -0,0 +1 @@
+langchain==0.0.259

examples/langchain-typescript-simple/README.md

@@ -0,0 +1,21 @@
+# LangChain
+
+This example is a basic "hello world" of using LangChain with Ollama using Node.js and Typescript.
+
+## Setup
+
+```shell
+npm install
+```
+
+## Run
+
+```shell
+ts-node main.ts
+```
+
+Running this example will print the response for "hello":
+
+```plaintext
+Hello! It's nice to meet you. hopefully you are having a great day! Is there something I can help you with or would you like to chat?
+```

examples/langchain-typescript-simple/main.ts

@@ -0,0 +1,15 @@
+import { Ollama} from 'langchain/llms/ollama';
+
+async function main() {
+  const ollama = new Ollama({
+    model: 'mistral'    
+    // other parameters can be found at https://js.langchain.com/docs/api/llms_ollama/classes/Ollama
+  })
+  const stream = await ollama.stream("Hello");
+
+  for await (const chunk of stream) {
+    process.stdout.write(chunk);
+  }
+}
+
+main();

examples/langchain-typescript-simple/package-lock.json

@@ -0,0 +1,997 @@
+{
+  "name": "with-langchain-typescript-simplegenerate",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "dependencies": {
+        "langchain": "^0.0.165"
+      },
+      "devDependencies": {
+        "typescript": "^5.2.2"
+      }
+    },
+    "node_modules/@anthropic-ai/sdk": {
+      "version": "0.6.2",
+      "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.6.2.tgz",
+      "integrity": "sha512-fB9PUj9RFT+XjkL+E9Ol864ZIJi+1P8WnbHspN3N3/GK2uSzjd0cbVIKTGgf4v3N8MwaQu+UWnU7C4BG/fap/g==",
+      "dependencies": {
+        "@types/node": "^18.11.18",
+        "@types/node-fetch": "^2.6.4",
+        "abort-controller": "^3.0.0",
+        "agentkeepalive": "^4.2.1",
+        "digest-fetch": "^1.3.0",
+        "form-data-encoder": "1.7.2",
+        "formdata-node": "^4.3.2",
+        "node-fetch": "^2.6.7"
+      }
+    },
+    "node_modules/@types/node": {
+      "version": "18.18.4",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.18.4.tgz",
+      "integrity": "sha512-t3rNFBgJRugIhackit2mVcLfF6IRc0JE4oeizPQL8Zrm8n2WY/0wOdpOPhdtG0V9Q2TlW/axbF1MJ6z+Yj/kKQ=="
+    },
+    "node_modules/@types/node-fetch": {
+      "version": "2.6.6",
+      "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.6.tgz",
+      "integrity": "sha512-95X8guJYhfqiuVVhRFxVQcf4hW/2bCuoPwDasMf/531STFoNoWTT7YDnWdXHEZKqAGUigmpG31r2FE70LwnzJw==",
+      "dependencies": {
+        "@types/node": "*",
+        "form-data": "^4.0.0"
+      }
+    },
+    "node_modules/@types/retry": {
+      "version": "0.12.0",
+      "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz",
+      "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA=="
+    },
+    "node_modules/@types/uuid": {
+      "version": "9.0.5",
+      "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.5.tgz",
+      "integrity": "sha512-xfHdwa1FMJ082prjSJpoEI57GZITiQz10r3vEJCHa2khEFQjKy91aWKz6+zybzssCvXUwE1LQWgWVwZ4nYUvHQ=="
+    },
+    "node_modules/abort-controller": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
+      "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
+      "dependencies": {
+        "event-target-shim": "^5.0.0"
+      },
+      "engines": {
+        "node": ">=6.5"
+      }
+    },
+    "node_modules/agentkeepalive": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz",
+      "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==",
+      "dependencies": {
+        "humanize-ms": "^1.2.1"
+      },
+      "engines": {
+        "node": ">= 8.0.0"
+      }
+    },
+    "node_modules/ansi-styles": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz",
+      "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==",
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
+    "node_modules/argparse": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
+      "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="
+    },
+    "node_modules/asynckit": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
+    },
+    "node_modules/base-64": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz",
+      "integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA=="
+    },
+    "node_modules/base64-js": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
+      "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ]
+    },
+    "node_modules/binary-extensions": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
+      "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/binary-search": {
+      "version": "1.3.6",
+      "resolved": "https://registry.npmjs.org/binary-search/-/binary-search-1.3.6.tgz",
+      "integrity": "sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA=="
+    },
+    "node_modules/camelcase": {
+      "version": "6.3.0",
+      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz",
+      "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==",
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/charenc": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
+      "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==",
+      "engines": {
+        "node": "*"
+      }
+    },
+    "node_modules/combined-stream": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+      "dependencies": {
+        "delayed-stream": "~1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
+    "node_modules/commander": {
+      "version": "10.0.1",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz",
+      "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==",
+      "engines": {
+        "node": ">=14"
+      }
+    },
+    "node_modules/crypt": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
+      "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==",
+      "engines": {
+        "node": "*"
+      }
+    },
+    "node_modules/decamelize": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",
+      "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/delayed-stream": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
+      "engines": {
+        "node": ">=0.4.0"
+      }
+    },
+    "node_modules/digest-fetch": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/digest-fetch/-/digest-fetch-1.3.0.tgz",
+      "integrity": "sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==",
+      "dependencies": {
+        "base-64": "^0.1.0",
+        "md5": "^2.3.0"
+      }
+    },
+    "node_modules/event-target-shim": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
+      "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/eventemitter3": {
+      "version": "4.0.7",
+      "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz",
+      "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw=="
+    },
+    "node_modules/expr-eval": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/expr-eval/-/expr-eval-2.0.2.tgz",
+      "integrity": "sha512-4EMSHGOPSwAfBiibw3ndnP0AvjDWLsMvGOvWEZ2F96IGk0bIVdjQisOHxReSkE13mHcfbuCiXw+G4y0zv6N8Eg=="
+    },
+    "node_modules/flat": {
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz",
+      "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==",
+      "bin": {
+        "flat": "cli.js"
+      }
+    },
+    "node_modules/form-data": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
+      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+      "dependencies": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "mime-types": "^2.1.12"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
+    "node_modules/form-data-encoder": {
+      "version": "1.7.2",
+      "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz",
+      "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A=="
+    },
+    "node_modules/formdata-node": {
+      "version": "4.4.1",
+      "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz",
+      "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==",
+      "dependencies": {
+        "node-domexception": "1.0.0",
+        "web-streams-polyfill": "4.0.0-beta.3"
+      },
+      "engines": {
+        "node": ">= 12.20"
+      }
+    },
+    "node_modules/humanize-ms": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
+      "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==",
+      "dependencies": {
+        "ms": "^2.0.0"
+      }
+    },
+    "node_modules/is-any-array": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/is-any-array/-/is-any-array-2.0.1.tgz",
+      "integrity": "sha512-UtilS7hLRu++wb/WBAw9bNuP1Eg04Ivn1vERJck8zJthEvXCBEBpGR/33u/xLKWEQf95803oalHrVDptcAvFdQ=="
+    },
+    "node_modules/is-buffer": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
+      "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
+    },
+    "node_modules/js-tiktoken": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.7.tgz",
+      "integrity": "sha512-biba8u/clw7iesNEWLOLwrNGoBP2lA+hTaBLs/D45pJdUPFXyxD6nhcDVtADChghv4GgyAiMKYMiRx7x6h7Biw==",
+      "dependencies": {
+        "base64-js": "^1.5.1"
+      }
+    },
+    "node_modules/js-yaml": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
+      "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
+      "dependencies": {
+        "argparse": "^2.0.1"
+      },
+      "bin": {
+        "js-yaml": "bin/js-yaml.js"
+      }
+    },
+    "node_modules/jsonpointer": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz",
+      "integrity": "sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/langchain": {
+      "version": "0.0.165",
+      "resolved": "https://registry.npmjs.org/langchain/-/langchain-0.0.165.tgz",
+      "integrity": "sha512-CpbNpjwaE+9lzjdw+pZz0VgnRrFivEgr7CVp9dDaAb5JpaJAA4V2v6uQ9ZPN+TSqupTQ79HFn2sfyZVEl2EG7Q==",
+      "dependencies": {
+        "@anthropic-ai/sdk": "^0.6.2",
+        "ansi-styles": "^5.0.0",
+        "binary-extensions": "^2.2.0",
+        "camelcase": "6",
+        "decamelize": "^1.2.0",
+        "expr-eval": "^2.0.2",
+        "flat": "^5.0.2",
+        "js-tiktoken": "^1.0.7",
+        "js-yaml": "^4.1.0",
+        "jsonpointer": "^5.0.1",
+        "langchainhub": "~0.0.6",
+        "langsmith": "~0.0.31",
+        "ml-distance": "^4.0.0",
+        "object-hash": "^3.0.0",
+        "openai": "~4.4.0",
+        "openapi-types": "^12.1.3",
+        "p-queue": "^6.6.2",
+        "p-retry": "4",
+        "uuid": "^9.0.0",
+        "yaml": "^2.2.1",
+        "zod": "^3.22.3",
+        "zod-to-json-schema": "^3.20.4"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "peerDependencies": {
+        "@aws-crypto/sha256-js": "^5.0.0",
+        "@aws-sdk/client-bedrock-runtime": "^3.422.0",
+        "@aws-sdk/client-dynamodb": "^3.310.0",
+        "@aws-sdk/client-kendra": "^3.352.0",
+        "@aws-sdk/client-lambda": "^3.310.0",
+        "@aws-sdk/client-s3": "^3.310.0",
+        "@aws-sdk/client-sagemaker-runtime": "^3.310.0",
+        "@aws-sdk/client-sfn": "^3.310.0",
+        "@aws-sdk/credential-provider-node": "^3.388.0",
+        "@azure/storage-blob": "^12.15.0",
+        "@clickhouse/client": "^0.0.14",
+        "@cloudflare/ai": "^1.0.12",
+        "@elastic/elasticsearch": "^8.4.0",
+        "@getmetal/metal-sdk": "*",
+        "@getzep/zep-js": "^0.7.0",
+        "@gomomento/sdk": "^1.23.0",
+        "@google-ai/generativelanguage": "^0.2.1",
+        "@google-cloud/storage": "^6.10.1",
+        "@huggingface/inference": "^1.5.1",
+        "@mozilla/readability": "*",
+        "@notionhq/client": "^2.2.10",
+        "@opensearch-project/opensearch": "*",
+        "@pinecone-database/pinecone": "^1.1.0",
+        "@planetscale/database": "^1.8.0",
+        "@qdrant/js-client-rest": "^1.2.0",
+        "@raycast/api": "^1.55.2",
+        "@smithy/eventstream-codec": "^2.0.5",
+        "@smithy/protocol-http": "^3.0.6",
+        "@smithy/signature-v4": "^2.0.10",
+        "@smithy/util-utf8": "^2.0.0",
+        "@supabase/postgrest-js": "^1.1.1",
+        "@supabase/supabase-js": "^2.10.0",
+        "@tensorflow-models/universal-sentence-encoder": "*",
+        "@tensorflow/tfjs-converter": "*",
+        "@tensorflow/tfjs-core": "*",
+        "@upstash/redis": "^1.20.6",
+        "@vercel/postgres": "^0.5.0",
+        "@writerai/writer-sdk": "^0.40.2",
+        "@xata.io/client": "^0.25.1",
+        "@xenova/transformers": "^2.5.4",
+        "@zilliz/milvus2-sdk-node": ">=2.2.7",
+        "apify-client": "^2.7.1",
+        "axios": "*",
+        "cassandra-driver": "^4.6.4",
+        "cheerio": "^1.0.0-rc.12",
+        "chromadb": "*",
+        "cohere-ai": ">=6.0.0",
+        "d3-dsv": "^2.0.0",
+        "epub2": "^3.0.1",
+        "faiss-node": "^0.3.0",
+        "fast-xml-parser": "^4.2.7",
+        "firebase-admin": "^11.9.0",
+        "google-auth-library": "^8.9.0",
+        "googleapis": "^126.0.1",
+        "hnswlib-node": "^1.4.2",
+        "html-to-text": "^9.0.5",
+        "ignore": "^5.2.0",
+        "ioredis": "^5.3.2",
+        "jsdom": "*",
+        "llmonitor": "*",
+        "lodash": "^4.17.21",
+        "mammoth": "*",
+        "mongodb": "^5.2.0",
+        "mysql2": "^3.3.3",
+        "neo4j-driver": "*",
+        "node-llama-cpp": "*",
+        "notion-to-md": "^3.1.0",
+        "pdf-parse": "1.1.1",
+        "peggy": "^3.0.2",
+        "pg": "^8.11.0",
+        "pg-copy-streams": "^6.0.5",
+        "pickleparser": "^0.1.0",
+        "playwright": "^1.32.1",
+        "portkey-ai": "^0.1.11",
+        "puppeteer": "^19.7.2",
+        "redis": "^4.6.4",
+        "replicate": "^0.18.0",
+        "sonix-speech-recognition": "^2.1.1",
+        "srt-parser-2": "^1.2.2",
+        "typeorm": "^0.3.12",
+        "typesense": "^1.5.3",
+        "usearch": "^1.1.1",
+        "vectordb": "^0.1.4",
+        "voy-search": "0.6.2",
+        "weaviate-ts-client": "^1.4.0",
+        "web-auth-library": "^1.0.3",
+        "youtube-transcript": "^1.0.6",
+        "youtubei.js": "^5.8.0"
+      },
+      "peerDependenciesMeta": {
+        "@aws-crypto/sha256-js": {
+          "optional": true
+        },
+        "@aws-sdk/client-bedrock-runtime": {
+          "optional": true
+        },
+        "@aws-sdk/client-dynamodb": {
+          "optional": true
+        },
+        "@aws-sdk/client-kendra": {
+          "optional": true
+        },
+        "@aws-sdk/client-lambda": {
+          "optional": true
+        },
+        "@aws-sdk/client-s3": {
+          "optional": true
+        },
+        "@aws-sdk/client-sagemaker-runtime": {
+          "optional": true
+        },
+        "@aws-sdk/client-sfn": {
+          "optional": true
+        },
+        "@aws-sdk/credential-provider-node": {
+          "optional": true
+        },
+        "@azure/storage-blob": {
+          "optional": true
+        },
+        "@clickhouse/client": {
+          "optional": true
+        },
+        "@cloudflare/ai": {
+          "optional": true
+        },
+        "@elastic/elasticsearch": {
+          "optional": true
+        },
+        "@getmetal/metal-sdk": {
+          "optional": true
+        },
+        "@getzep/zep-js": {
+          "optional": true
+        },
+        "@gomomento/sdk": {
+          "optional": true
+        },
+        "@google-ai/generativelanguage": {
+          "optional": true
+        },
+        "@google-cloud/storage": {
+          "optional": true
+        },
+        "@huggingface/inference": {
+          "optional": true
+        },
+        "@mozilla/readability": {
+          "optional": true
+        },
+        "@notionhq/client": {
+          "optional": true
+        },
+        "@opensearch-project/opensearch": {
+          "optional": true
+        },
+        "@pinecone-database/pinecone": {
+          "optional": true
+        },
+        "@planetscale/database": {
+          "optional": true
+        },
+        "@qdrant/js-client-rest": {
+          "optional": true
+        },
+        "@raycast/api": {
+          "optional": true
+        },
+        "@smithy/eventstream-codec": {
+          "optional": true
+        },
+        "@smithy/protocol-http": {
+          "optional": true
+        },
+        "@smithy/signature-v4": {
+          "optional": true
+        },
+        "@smithy/util-utf8": {
+          "optional": true
+        },
+        "@supabase/postgrest-js": {
+          "optional": true
+        },
+        "@supabase/supabase-js": {
+          "optional": true
+        },
+        "@tensorflow-models/universal-sentence-encoder": {
+          "optional": true
+        },
+        "@tensorflow/tfjs-converter": {
+          "optional": true
+        },
+        "@tensorflow/tfjs-core": {
+          "optional": true
+        },
+        "@upstash/redis": {
+          "optional": true
+        },
+        "@vercel/postgres": {
+          "optional": true
+        },
+        "@writerai/writer-sdk": {
+          "optional": true
+        },
+        "@xata.io/client": {
+          "optional": true
+        },
+        "@xenova/transformers": {
+          "optional": true
+        },
+        "@zilliz/milvus2-sdk-node": {
+          "optional": true
+        },
+        "apify-client": {
+          "optional": true
+        },
+        "axios": {
+          "optional": true
+        },
+        "cassandra-driver": {
+          "optional": true
+        },
+        "cheerio": {
+          "optional": true
+        },
+        "chromadb": {
+          "optional": true
+        },
+        "cohere-ai": {
+          "optional": true
+        },
+        "d3-dsv": {
+          "optional": true
+        },
+        "epub2": {
+          "optional": true
+        },
+        "faiss-node": {
+          "optional": true
+        },
+        "fast-xml-parser": {
+          "optional": true
+        },
+        "firebase-admin": {
+          "optional": true
+        },
+        "google-auth-library": {
+          "optional": true
+        },
+        "googleapis": {
+          "optional": true
+        },
+        "hnswlib-node": {
+          "optional": true
+        },
+        "html-to-text": {
+          "optional": true
+        },
+        "ignore": {
+          "optional": true
+        },
+        "ioredis": {
+          "optional": true
+        },
+        "jsdom": {
+          "optional": true
+        },
+        "llmonitor": {
+          "optional": true
+        },
+        "lodash": {
+          "optional": true
+        },
+        "mammoth": {
+          "optional": true
+        },
+        "mongodb": {
+          "optional": true
+        },
+        "mysql2": {
+          "optional": true
+        },
+        "neo4j-driver": {
+          "optional": true
+        },
+        "node-llama-cpp": {
+          "optional": true
+        },
+        "notion-to-md": {
+          "optional": true
+        },
+        "pdf-parse": {
+          "optional": true
+        },
+        "peggy": {
+          "optional": true
+        },
+        "pg": {
+          "optional": true
+        },
+        "pg-copy-streams": {
+          "optional": true
+        },
+        "pickleparser": {
+          "optional": true
+        },
+        "playwright": {
+          "optional": true
+        },
+        "portkey-ai": {
+          "optional": true
+        },
+        "puppeteer": {
+          "optional": true
+        },
+        "redis": {
+          "optional": true
+        },
+        "replicate": {
+          "optional": true
+        },
+        "sonix-speech-recognition": {
+          "optional": true
+        },
+        "srt-parser-2": {
+          "optional": true
+        },
+        "typeorm": {
+          "optional": true
+        },
+        "typesense": {
+          "optional": true
+        },
+        "usearch": {
+          "optional": true
+        },
+        "vectordb": {
+          "optional": true
+        },
+        "voy-search": {
+          "optional": true
+        },
+        "weaviate-ts-client": {
+          "optional": true
+        },
+        "web-auth-library": {
+          "optional": true
+        },
+        "youtube-transcript": {
+          "optional": true
+        },
+        "youtubei.js": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/langchainhub": {
+      "version": "0.0.6",
+      "resolved": "https://registry.npmjs.org/langchainhub/-/langchainhub-0.0.6.tgz",
+      "integrity": "sha512-SW6105T+YP1cTe0yMf//7kyshCgvCTyFBMTgH2H3s9rTAR4e+78DA/BBrUL/Mt4Q5eMWui7iGuAYb3pgGsdQ9w=="
+    },
+    "node_modules/langsmith": {
+      "version": "0.0.42",
+      "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.0.42.tgz",
+      "integrity": "sha512-sFuN+e7E+pPBIRaRgFqZh/BRBWNHTZNAwi6uj4kydQawooCZYoJmM5snOkiQrhVSvAhgu6xFhLvmfvkPcKzD7w==",
+      "dependencies": {
+        "@types/uuid": "^9.0.1",
+        "commander": "^10.0.1",
+        "p-queue": "^6.6.2",
+        "p-retry": "4",
+        "uuid": "^9.0.0"
+      },
+      "bin": {
+        "langsmith": "dist/cli/main.cjs"
+      }
+    },
+    "node_modules/md5": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
+      "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==",
+      "dependencies": {
+        "charenc": "0.0.2",
+        "crypt": "0.0.2",
+        "is-buffer": "~1.1.6"
+      }
+    },
+    "node_modules/mime-db": {
+      "version": "1.52.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/mime-types": {
+      "version": "2.1.35",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+      "dependencies": {
+        "mime-db": "1.52.0"
+      },
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/ml-array-mean": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/ml-array-mean/-/ml-array-mean-1.1.6.tgz",
+      "integrity": "sha512-MIdf7Zc8HznwIisyiJGRH9tRigg3Yf4FldW8DxKxpCCv/g5CafTw0RRu51nojVEOXuCQC7DRVVu5c7XXO/5joQ==",
+      "dependencies": {
+        "ml-array-sum": "^1.1.6"
+      }
+    },
+    "node_modules/ml-array-sum": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/ml-array-sum/-/ml-array-sum-1.1.6.tgz",
+      "integrity": "sha512-29mAh2GwH7ZmiRnup4UyibQZB9+ZLyMShvt4cH4eTK+cL2oEMIZFnSyB3SS8MlsTh6q/w/yh48KmqLxmovN4Dw==",
+      "dependencies": {
+        "is-any-array": "^2.0.0"
+      }
+    },
+    "node_modules/ml-distance": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/ml-distance/-/ml-distance-4.0.1.tgz",
+      "integrity": "sha512-feZ5ziXs01zhyFUUUeZV5hwc0f5JW0Sh0ckU1koZe/wdVkJdGxcP06KNQuF0WBTj8FttQUzcvQcpcrOp/XrlEw==",
+      "dependencies": {
+        "ml-array-mean": "^1.1.6",
+        "ml-distance-euclidean": "^2.0.0",
+        "ml-tree-similarity": "^1.0.0"
+      }
+    },
+    "node_modules/ml-distance-euclidean": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/ml-distance-euclidean/-/ml-distance-euclidean-2.0.0.tgz",
+      "integrity": "sha512-yC9/2o8QF0A3m/0IXqCTXCzz2pNEzvmcE/9HFKOZGnTjatvBbsn4lWYJkxENkA4Ug2fnYl7PXQxnPi21sgMy/Q=="
+    },
+    "node_modules/ml-tree-similarity": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/ml-tree-similarity/-/ml-tree-similarity-1.0.0.tgz",
+      "integrity": "sha512-XJUyYqjSuUQkNQHMscr6tcjldsOoAekxADTplt40QKfwW6nd++1wHWV9AArl0Zvw/TIHgNaZZNvr8QGvE8wLRg==",
+      "dependencies": {
+        "binary-search": "^1.3.5",
+        "num-sort": "^2.0.0"
+      }
+    },
+    "node_modules/ms": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
+    },
+    "node_modules/node-domexception": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
+      "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/jimmywarting"
+        },
+        {
+          "type": "github",
+          "url": "https://paypal.me/jimmywarting"
+        }
+      ],
+      "engines": {
+        "node": ">=10.5.0"
+      }
+    },
+    "node_modules/node-fetch": {
+      "version": "2.7.0",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
+      "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
+      "dependencies": {
+        "whatwg-url": "^5.0.0"
+      },
+      "engines": {
+        "node": "4.x || >=6.0.0"
+      },
+      "peerDependencies": {
+        "encoding": "^0.1.0"
+      },
+      "peerDependenciesMeta": {
+        "encoding": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/num-sort": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/num-sort/-/num-sort-2.1.0.tgz",
+      "integrity": "sha512-1MQz1Ed8z2yckoBeSfkQHHO9K1yDRxxtotKSJ9yvcTUUxSvfvzEq5GwBrjjHEpMlq/k5gvXdmJ1SbYxWtpNoVg==",
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/object-hash": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz",
+      "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==",
+      "engines": {
+        "node": ">= 6"
+      }
+    },
+    "node_modules/openai": {
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/openai/-/openai-4.4.0.tgz",
+      "integrity": "sha512-JN0t628Kh95T0IrXl0HdBqnlJg+4Vq0Bnh55tio+dfCnyzHvMLiWyCM9m726MAJD2YkDU4/8RQB6rNbEq9ct2w==",
+      "dependencies": {
+        "@types/node": "^18.11.18",
+        "@types/node-fetch": "^2.6.4",
+        "abort-controller": "^3.0.0",
+        "agentkeepalive": "^4.2.1",
+        "digest-fetch": "^1.3.0",
+        "form-data-encoder": "1.7.2",
+        "formdata-node": "^4.3.2",
+        "node-fetch": "^2.6.7"
+      },
+      "bin": {
+        "openai": "bin/cli"
+      }
+    },
+    "node_modules/openapi-types": {
+      "version": "12.1.3",
+      "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz",
+      "integrity": "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw=="
+    },
+    "node_modules/p-finally": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
+      "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==",
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/p-queue": {
+      "version": "6.6.2",
+      "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz",
+      "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==",
+      "dependencies": {
+        "eventemitter3": "^4.0.4",
+        "p-timeout": "^3.2.0"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/p-retry": {
+      "version": "4.6.2",
+      "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz",
+      "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==",
+      "dependencies": {
+        "@types/retry": "0.12.0",
+        "retry": "^0.13.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/p-timeout": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz",
+      "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==",
+      "dependencies": {
+        "p-finally": "^1.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/retry": {
+      "version": "0.13.1",
+      "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz",
+      "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==",
+      "engines": {
+        "node": ">= 4"
+      }
+    },
+    "node_modules/tr46": {
+      "version": "0.0.3",
+      "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
+      "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
+    },
+    "node_modules/typescript": {
+      "version": "5.2.2",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz",
+      "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==",
+      "dev": true,
+      "bin": {
+        "tsc": "bin/tsc",
+        "tsserver": "bin/tsserver"
+      },
+      "engines": {
+        "node": ">=14.17"
+      }
+    },
+    "node_modules/uuid": {
+      "version": "9.0.1",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz",
+      "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==",
+      "funding": [
+        "https://github.com/sponsors/broofa",
+        "https://github.com/sponsors/ctavan"
+      ],
+      "bin": {
+        "uuid": "dist/bin/uuid"
+      }
+    },
+    "node_modules/web-streams-polyfill": {
+      "version": "4.0.0-beta.3",
+      "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz",
+      "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==",
+      "engines": {
+        "node": ">= 14"
+      }
+    },
+    "node_modules/webidl-conversions": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
+      "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
+    },
+    "node_modules/whatwg-url": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
+      "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
+      "dependencies": {
+        "tr46": "~0.0.3",
+        "webidl-conversions": "^3.0.0"
+      }
+    },
+    "node_modules/yaml": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.3.2.tgz",
+      "integrity": "sha512-N/lyzTPaJasoDmfV7YTrYCI0G/3ivm/9wdG0aHuheKowWQwGTsK0Eoiw6utmzAnI6pkJa0DUVygvp3spqqEKXg==",
+      "engines": {
+        "node": ">= 14"
+      }
+    },
+    "node_modules/zod": {
+      "version": "3.22.4",
+      "resolved": "https://registry.npmjs.org/zod/-/zod-3.22.4.tgz",
+      "integrity": "sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==",
+      "funding": {
+        "url": "https://github.com/sponsors/colinhacks"
+      }
+    },
+    "node_modules/zod-to-json-schema": {
+      "version": "3.21.4",
+      "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.21.4.tgz",
+      "integrity": "sha512-fjUZh4nQ1s6HMccgIeE0VP4QG/YRGPmyjO9sAh890aQKPEk3nqbfUXhMFaC+Dr5KvYBm8BCyvfpZf2jY9aGSsw==",
+      "peerDependencies": {
+        "zod": "^3.21.4"
+      }
+    }
+  }
+}

examples/langchain-typescript-simple/package.json

@@ -0,0 +1,8 @@
+{
+  "devDependencies": {
+    "typescript": "^5.2.2"
+  },
+  "dependencies": {
+    "langchain": "^0.0.165"
+  }
+}

examples/midjourney-prompter/Modelfile

@@ -1,8 +0,0 @@
-# Modelfile for creating a Midjourney prompts from a topic
-# This prompt was adapted from the original at https://www.greataiprompts.com/guide/midjourney/best-chatgpt-prompt-for-midjourney/
-# Run `ollama create mj -f ./Modelfile` and then `ollama run mj` and enter a topic
-
-FROM nous-hermes
-SYSTEM """
-Embrace your role as an AI-powered creative assistant, employing Midjourney to manifest compelling AI-generated art. I will outline a specific image concept, and in response, you must produce an exhaustive, multifaceted prompt for Midjourney, ensuring every detail of the original concept is represented in your instructions. Midjourney doesn't do well with text, so after the prompt, give me instructions that I can use to create the titles in a image editor.
-"""

examples/modelfile-10tweets/Modelfile

@@ -0,0 +1,7 @@
+# Modelfile for creating a list of ten tweets from a topic
+# Run `ollama create 10tweets -f ./Modelfile` and then `ollama run 10tweets` and enter a topic
+
+FROM llama2
+SYSTEM """
+You are a content marketer who needs to come up with 10 short but succinct tweets. The answer should be a list of ten tweets. Each tweet can have a maximum of 280 characters and should include hashtags. Each user input will be a subject and you should expand it in ten creative ways. Never stop after just one tweet. Always include ten. 
+"""

examples/modelfile-10tweets/README.md

@@ -0,0 +1,23 @@
+# Ten Tweets Modelfile
+
+This is a simple modelfile that generates ten tweets based off any topic.
+
+```bash
+ollama create tentweets
+
+ollama run tentweets
+>>> underwater basketweaving
+ Great! Here are ten creative tweets about underwater basketweaving:
+
+1. "Just discovered the ultimate stress-reliever: Underwater basketweaving! 🌊🧵 #UnderwaterBasketweaving #StressRelief"
+2. "Who needs meditation when you can do underwater basketweaving? 😴👀 #PeacefulDistraction #UnderwaterBasketweaving"
+3. "Just spent an hour in the pool and still managed to knot my basket. Goal: untangle it before next session. 💪🏽 #ChallengeAccepted #UnderwaterBasketweaving"
+4. "When life gives you lemons, make underwater basketweaving! 🍋🧵 #LemonadeLife #UnderwaterBasketweaving"
+5. "Just realized my underwater basketweaving skills could come in handy during a zombie apocalypse. 😂🧡 #SurvivalTips #UnderwaterBasketweaving"
+6. "I'm not lazy, I'm just conserving energy for my next underwater basketweaving session. 😴💤 #LazyDay #UnderwaterBasketweaving"
+7. "Just found my inner peace while doing underwater basketweaving. It's like meditation, but with knots! 🙏🧵 #Mindfulness #UnderwaterBasketweaving"
+8. "Why study for exams when you can do underwater basketweaving and forget all your worries? 😜🧵 #ProcrastinationStation #UnderwaterBasketweaving"
+9. "Just had to cut my underwater basketweaving session short due to a sudden urge to breathe. 🤯🌊 #AquaticAdventures #UnderwaterBasketweaving"
+10. "I'm not sure what's more impressive: my underwater basketweaving skills or the fact that I didn't drown trying to make this tweet. 😅🧵 #Accomplishment 
+#UnderwaterBasketweaving"
+```

examples/modelfile-devopsengineer/Modelfile

@@ -0,0 +1,8 @@
+# Modelfile for creating a devops engineer assistant
+# Run `ollama create devops-engineer -f ./Modelfile` and then `ollama run devops-engineer` and enter a topic
+
+FROM mistral
+PARAMETER temperature 1
+SYSTEM """
+You are a senior devops engineer, acting as an assistant. You offer help with cloud technologies like: Terraform, AWS, kubernetes, python. You answer with code examples when possible
+"""

examples/modelfile-devopsengineer/README.md

@@ -0,0 +1,40 @@
+# DevOps Engineer Modelfile
+
+This is a simple modelfile that takes will respond to any questions that a senior DevOps engineer should know how to solve.
+
+````text
+ollama create devopsengineer
+
+ollama run devopsengineer
+>>> how can i proxy all connections from inside the office through a single server that caches large files. Provide a config file for nginx to achieve this?
+
+Here is an example configuration file for Nginx that can be used to proxy all connections from inside the office network to a centralized cache server:
+```perl
+http {
+    upstream cache_server {
+        server <cache_server_ip>;
+        server <cache_server_ip>;
+        server <cache_server_ip>;
+    }
+    
+    server {
+        listen 80;
+        server_name localhost;
+        
+        location / {
+            proxy_pass http://cache_server;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+    }
+}
+```
+In this configuration, we define an `upstream` block that lists the IP addresses of the cache servers. We then define a server block that listens on port 80 and routes all 
+traffic to the `/` location to the cache servers using the `proxy_pass` directive. The `proxy_set_header` directives are used to preserve the source IP address of the client
+request when forwarding it to the cache server.
+
+To use this configuration, you would need to replace the placeholder `<cache_server_ip>` with the actual IP addresses of your cache servers. You would also need to make sure
+that the cache servers are configured to accept incoming connections from the Nginx server and handle requests for files.
+
+````

examples/modelfile-midjourney/Modelfile

@@ -0,0 +1,11 @@
+# Modelfile for creating a Midjourney prompts from a topic
+# This prompt was adapted from the original at https://www.greataiprompts.com/guide/midjourney/best-chatgpt-prompt-for-midjourney/
+# Run `ollama create mj -f ./Modelfile` and then `ollama run mj` and enter a topic
+
+FROM zephyr
+PARAMETER temperature 0.8
+PARAMETER top_k 500
+PARAMETER top_p 0.9
+SYSTEM """
+Embrace your role as a creative illustrator. Based on a concept provided, you must produce a single paragraph with a multifaceted description of an image, ensuring significant details of the concept and more is represented in your instructions. You do not need to write complete sentences but rather short concepts with the following information: the level of detail that should be represented, an artistic style and maybe a specific name of a painter or illustrator, the ideal color pallete, lighting, mood, perspective, the setting, time of day, weather, the season, the time period, location, materials, the textures, patterns, lines, brushstrokes, techniques, the medium, the genre, the rendering style. Don't include everything and keep the description length under 250 words. 
+"""

examples/modelfile-midjourney/README.md

@@ -0,0 +1,11 @@
+# Midjourney Prompt Generator Modelfile
+
+This simple modelfile will help create a prompt to feed to Midjourney.
+
+```text
+ollama create midjourney
+
+ollama run midjourney
+>>> a sports car in the mountains. 
+A sleek, high-performance automobile cuts through a serpentine mountain landscape. The concept is a classic illustration of speed and power, depicted in the style of pop art by Andy Warhol. The color palette is dominated by bold, primary hues of red, blue, and yellow, with striking accent colors of white, black, and metallic shades. The lighting is bright and focused, casting sharp shadows on the rugged terrain. A sense of excitement and anticipation permeates throughout the scene, as the car navigates a treacherous course through the winding road. The perspective is low, allowing for a full view of the vehicle's sleek lines and intricate details. The setting takes place in the afternoon during a sunny day in autumn, as evidenced by the vibrant foliage on the mountainside. The time period is modern, with nods to classic car design. The materials are primarily digital, allowing for smooth curves and sharp contrasts. The textures are sleek and polished, with meticulously detailed lines and brushstrokes that accentuate the car's aerodynamic design. The patterns consist of geometric shapes and bold stripes, adding to the car's dynamic appeal. The genre is modern realism, with a focus on precision and detail. The rendering style is highly technical, capturing the nuances and subtleties of the vehicle and its surroundings in breathtaking detail.
+```

examples/modelfile-recipemaker/README.md

@@ -0,0 +1,20 @@
+# Recipe Maker Modelfile 
+
+Simple modelfile to generate a recipe from a short list of ingredients.
+
+```
+ollama create recipemaker
+
+ollama run recipemaker
+>>> chilli pepper, white chocolate, kale
+ Ingredients:
+- 1 small chili pepper
+- 4 squares of white chocolate
+- handful of kale leaves
+
+Instructions:
+1. In a blender or food processor, puree the chilies and white chocolate until smooth.
+2. Add the chopped kale leaves to the blender and pulse until well combined.
+3. Serve immediately as a dip for crackers or use it as an ingredient in your favorite recipe. The mixture of spicy chili pepper with sweet white chocolate and nutritious 
+kale will make your taste buds dance with delight!
+```

examples/modelfile-sentiments/Modelfile

@@ -0,0 +1,28 @@
+# Modelfile for creating a sentiment analyzer. 
+# Run `ollama create sentiments -f pathtofile` and then `ollama run sentiments` and enter a topic
+
+FROM orca
+TEMPLATE """
+{{- if .First }}
+### System:
+{{ .System }}
+{{- end }}
+### User: 
+I hate it when my phone dies
+### Response: 
+NEGATIVE
+### User: 
+He is awesome
+### Response: 
+POSITIVE
+### User: 
+This is the link to the article
+### Response: 
+NEUTRAL
+### User:
+{{ .Prompt }}
+
+### Response:
+"""
+
+SYSTEM """You are a sentiment analyzer. You will receive text and output only one word, either POSITIVE or NEGATIVE or NEUTRAL, depending on the sentiment of the text."""

examples/modelfile-sentiments/Readme.md

@@ -0,0 +1,25 @@
+# Sentiments Modelfile
+
+This is a simple sentiments analyzer using the Orca model. When you pull Orca from the registry, it has a Template already defined that looks like this:
+
+```Modelfile
+{{- if .First }}
+### System:
+{{ .System }}
+{{- end }}
+
+### User:
+{{ .Prompt }}
+
+### Response:
+```
+
+If we just wanted to have the text:
+
+```Plaintext
+You are a sentiment analyzer. You will receive text and output only one word, either POSITIVE or NEGATIVE or NEUTRAL, depending on the sentiment of the text.
+```
+
+then we could have put this in a SYSTEM block. But we want to provide examples which require updating the full Template. Any Modelfile you create will inherit all the settings from the source model. But in this example, we are overriding the Template.
+
+When providing examples for the input and output, you should include the way the model usually provides information. Since the Orca model expects a user prompt to appear after ### User: and the response is after ### Response, we should format our examples like that as well. If we were using the Llama 2 model, the format would be a bit different.

examples/modelfile-tweetwriter/Modelfile

@@ -3,5 +3,5 @@
 
 FROM nous-hermes
 SYSTEM """
-You are a content marketer who needs to come up with a short but succinct tweet. Make sure to include the appropriate hashtags and links. Sometimes when appropriate, describe a meme that can be includes as well. All answers should be in the form of a tweet which has a max size of 280 characters. Every instruction will be the topic to create a tweet about.
+You are a content marketer who needs to come up with a short but succinct tweet. Make sure to include the appropriate hashtags and links. Sometimes when appropriate, describe a meme that can be included as well. All answers should be in the form of a tweet which has a max size of 280 characters. Every instruction will be the topic to create a tweet about.
 """

examples/python-dockerit/Modelfile

@@ -0,0 +1,20 @@
+FROM mistral
+SYSTEM """
+You are an experienced Devops engineer focused on docker. When given specifications for a particular need or application you know the best way to host that within a docker container. For instance if someone tells you they want an nginx server to host files located at /web you will answer as follows
+
+---start
+FROM nginx:alpine
+COPY /myweb /usr/share/nginx/html
+EXPOSE 80
+---end
+
+Notice that the answer you should give is just the contents of the dockerfile with no explanation and there are three dashes and the word start at the beginning and 3 dashes and the word end. The full output can be piped into a file and run as is. Here is another example. The user will ask to launch a Postgres server with a password of abc123. And the response should be
+
+---start
+FROM postgres:latest
+ENV POSTGRES_PASSWORD=abc123
+EXPOSE 5432
+---end
+
+Again it's just the contents of the dockerfile and nothing else.
+"""

examples/python-dockerit/README.md

@@ -0,0 +1,15 @@
+# DockerIt
+
+DockerIt is a tool to help you build and run your application in a Docker container. It consists of a model that defines the system prompt and model weights to use, along with a python script to then build the container and run the image automatically. 
+
+## Caveats
+
+This is an simple example. It's assuming the Dockerfile content generated is going to work. In many cases, even with simple web servers, it fails when trying to copy files that don't exist. It's simply an example of what you could possibly do.
+
+## Example Usage
+
+```bash
+> python3 ./dockerit.py "simple postgres server with admin password set to 123"
+Enter the name of the image: matttest
+Container named happy_keller  started with id:  7c201bb6c30f02b356ddbc8e2a5af9d7d7d7b8c228519c9a501d15c0bd9d6b3e
+```

examples/python-dockerit/dockerit.py

@@ -0,0 +1,17 @@
+import requests, json, docker, io, sys
+inputDescription = " ".join(sys.argv[1:])
+imageName = input("Enter the name of the image: ")
+client = docker.from_env()
+s = requests.Session()
+output=""
+with s.post('http://localhost:11434/api/generate', json={'model': 'dockerit', 'prompt': inputDescription}, stream=True) as r:
+  for line in r.iter_lines():
+    if line:
+      j = json.loads(line)
+      if "response" in j:
+        output = output +j["response"]
+output = output[output.find("---start")+9:output.find("---end")-1]
+f = io.BytesIO(bytes(output, 'utf-8'))
+client.images.build(fileobj=f, tag=imageName)
+container = client.containers.run(imageName, detach=True)
+print("Container named", container.name, " started with id: ",container.id)

examples/python-dockerit/requirements.txt

@@ -0,0 +1 @@
+docker

examples/python-rag-newssummary/README.md

@@ -0,0 +1,22 @@
+# News Summarizer
+
+This example goes through a series of steps:
+
+  1. You choose a topic area (e.g., "news", "NVidia", "music", etc.).
+  2. Gets the most recent articles on that topic from various sources.
+  3. Uses Ollama to summarize each article.
+  4. Creates chunks of sentences from each article.
+  5. Uses Sentence Transformers to generate embeddings for each of those chunks.
+  6. You enter a question regarding the summaries shown.
+  7. Uses Sentence Transformers to generate an embedding for that question.
+  8. Uses the embedded question to find the most similar chunks.
+  9. Feeds all that to Ollama to generate a good answer to your question based on these news articles.
+
+This example lets you pick from a few different topic areas, then summarize the most recent x articles for that topic. It then creates chunks of sentences from each article and then generates embeddings for each of those chunks.
+
+You can run the example like this:
+
+```bash
+pip install -r requirements.txt
+python summ.py
+```

examples/python-rag-newssummary/requirements.txt

@@ -0,0 +1,9 @@
+beautifulsoup4==4.12.2
+feedparser==6.0.10
+mattsollamatools==0.0.8
+newspaper3k==0.2.8
+nltk==3.8.1
+numpy==1.24.3
+Requests==2.31.0
+scikit_learn==1.3.0
+sentence_transformers==2.2.2

examples/python-rag-newssummary/summ.py

@@ -0,0 +1,86 @@
+import curses
+import json
+from utils import get_url_for_topic, topic_urls, menu, getUrls, get_summary, getArticleText, knn_search
+import requests
+from sentence_transformers import SentenceTransformer
+from mattsollamatools import chunker
+
+if __name__ == "__main__":
+    chosen_topic = curses.wrapper(menu)
+    print("Here is your news summary:\n")
+    urls = getUrls(chosen_topic, n=5)
+    model = SentenceTransformer('all-MiniLM-L6-v2')
+    allEmbeddings = []
+
+    for url in urls:
+      article={}
+      article['embeddings'] = []
+      article['url'] = url
+      text = getArticleText(url)
+      summary = get_summary(text)
+      chunks = chunker(text)  # Use the chunk_text function from web_utils
+      embeddings = model.encode(chunks)
+      for (chunk, embedding) in zip(chunks, embeddings):
+        item = {}
+        item['source'] = chunk
+        item['embedding'] = embedding.tolist()  # Convert NumPy array to list
+        item['sourcelength'] = len(chunk)
+        article['embeddings'].append(item)
+    
+      allEmbeddings.append(article)
+
+      print(f"{summary}\n")
+
+    
+    while True:
+      context = []
+      # Input a question from the user
+      question = input("Enter your question about the news, or type quit: ")
+
+      if question.lower() == 'quit':
+        break
+
+      # Embed the user's question
+      question_embedding = model.encode([question])
+
+      # Perform KNN search to find the best matches (indices and source text)
+      best_matches = knn_search(question_embedding, allEmbeddings, k=10)
+
+
+      sourcetext=""
+      for i, (index, source_text) in enumerate(best_matches, start=1):
+          sourcetext += f"{i}. Index: {index}, Source Text: {source_text}"
+
+      systemPrompt = f"Only use the following information to answer the question. Do not use anything else: {sourcetext}"
+
+      url = "http://localhost:11434/api/generate"
+
+      payload = {
+      "model": "mistral-openorca",
+      "prompt": question, 
+      "system": systemPrompt,
+      "stream": False, 
+      "context": context
+      }
+
+      # Convert the payload to a JSON string
+      payload_json = json.dumps(payload)
+
+      # Set the headers to specify JSON content
+      headers = {
+          "Content-Type": "application/json"
+      }
+
+      # Send the POST request
+      response = requests.post(url, data=payload_json, headers=headers)
+
+      # Check the response
+      if response.status_code == 200:
+          output = json.loads(response.text)
+          context = output['context']
+          print(output['response']+ "\n")
+          
+
+      else:
+          print(f"Request failed with status code {response.status_code}")
+

examples/python-rag-newssummary/utils.py

@@ -0,0 +1,108 @@
+import curses
+import feedparser
+import requests
+import unicodedata
+import json
+from newspaper import Article
+from bs4 import BeautifulSoup
+from nltk.tokenize import sent_tokenize, word_tokenize
+import numpy as np
+from sklearn.neighbors import NearestNeighbors
+from mattsollamatools import chunker
+
+# Create a dictionary to store topics and their URLs
+topic_urls = {
+    "Mac": "https://9to5mac.com/guides/mac/feed",
+    "News": "http://www.npr.org/rss/rss.php?id=1001",
+    "Nvidia": "https://nvidianews.nvidia.com/releases.xml",
+    "Raspberry Pi": "https://www.raspberrypi.com/news/feed/", 
+    "Music": "https://www.billboard.com/c/music/music-news/feed/"
+}
+
+# Use curses to create a menu of topics
+def menu(stdscr):
+    chosen_topic = get_url_for_topic(stdscr)  
+    url = topic_urls[chosen_topic] if chosen_topic in topic_urls else "Topic not found"
+    
+    stdscr.addstr(len(topic_urls) + 3, 0, f"Selected URL for {chosen_topic}: {url}")
+    stdscr.refresh()
+    
+    return chosen_topic
+
+# You have chosen a topic. Now return the url for that topic
+def get_url_for_topic(stdscr):
+    curses.curs_set(0)  # Hide the cursor
+    stdscr.clear()
+
+    stdscr.addstr(0, 0, "Choose a topic using the arrow keys (Press Enter to select):")
+
+    # Create a list of topics
+    topics = list(topic_urls.keys())
+    current_topic = 0
+
+    while True:
+        for i, topic in enumerate(topics):
+            if i == current_topic:
+                stdscr.addstr(i + 2, 2, f"> {topic}")
+            else:
+                stdscr.addstr(i + 2, 2, f"  {topic}")
+
+        stdscr.refresh()
+
+        key = stdscr.getch()
+
+        if key == curses.KEY_DOWN and current_topic < len(topics) - 1:
+            current_topic += 1
+        elif key == curses.KEY_UP and current_topic > 0:
+            current_topic -= 1
+        elif key == 10:  # Enter key
+            return topic_urls[topics[current_topic]]
+
+# Get the last N URLs from an RSS feed
+def getUrls(feed_url, n=20):
+    feed = feedparser.parse(feed_url)
+    entries = feed.entries[-n:]
+    urls = [entry.link for entry in entries]
+    return urls
+
+# Often there are a bunch of ads and menus on pages for a news article. This uses newspaper3k to get just the text of just the article.
+def getArticleText(url):
+  article = Article(url)
+  article.download()
+  article.parse()
+  return article.text
+
+def get_summary(text):
+  systemPrompt = "Write a concise summary of the text, return your responses with 5 lines that cover the key points of the text given."
+  prompt = text
+  
+  url = "http://localhost:11434/api/generate"
+
+  payload = {
+    "model": "mistral-openorca",
+    "prompt": prompt, 
+    "system": systemPrompt,
+    "stream": False
+  }
+  payload_json = json.dumps(payload)
+  headers = {"Content-Type": "application/json"}
+  response = requests.post(url, data=payload_json, headers=headers)
+
+  return json.loads(response.text)["response"]
+
+# Perform K-nearest neighbors (KNN) search
+def knn_search(question_embedding, embeddings, k=5):
+    X = np.array([item['embedding'] for article in embeddings for item in article['embeddings']])
+    source_texts = [item['source'] for article in embeddings for item in article['embeddings']]
+    
+    # Fit a KNN model on the embeddings
+    knn = NearestNeighbors(n_neighbors=k, metric='cosine')
+    knn.fit(X)
+    
+    # Find the indices and distances of the k-nearest neighbors
+    distances, indices = knn.kneighbors(question_embedding, n_neighbors=k)
+    
+    # Get the indices and source texts of the best matches
+    best_matches = [(indices[0][i], source_texts[indices[0][i]]) for i in range(k)]
+    
+    return best_matches

examples/python-simplegenerate/client.py

@@ -0,0 +1,38 @@
+import json
+import requests
+
+# NOTE: ollama must be running for this to work, start the ollama app or run `ollama serve`
+model = 'llama2' # TODO: update this for whatever model you wish to use
+
+def generate(prompt, context):
+    r = requests.post('http://localhost:11434/api/generate',
+                      json={
+                          'model': model,
+                          'prompt': prompt,
+                          'context': context,
+                      },
+                      stream=True)
+    r.raise_for_status()
+
+    for line in r.iter_lines():
+        body = json.loads(line)
+        response_part = body.get('response', '')
+        # the response streams one token at a time, print that as we receive it
+        print(response_part, end='', flush=True)
+
+        if 'error' in body:
+            raise Exception(body['error'])
+
+        if body.get('done', False):
+            return body['context']
+
+def main():
+    context = [] # the context stores a conversation history, you can use this to make the model more context aware
+    while True:
+        user_input = input("Enter a prompt: ")
+        print()
+        context = generate(user_input, context)
+        print()
+
+if __name__ == "__main__":
+    main()

examples/typescript-mentors/.gitignore

@@ -0,0 +1,2 @@
+node_modules
+package-lock.json

examples/typescript-mentors/README.md

@@ -0,0 +1,21 @@
+# Ask the Mentors
+
+This example demonstrates how one would create a set of 'mentors' you can have a conversation with. The mentors are generated using the `character-generator.ts` file. This will use **Stable Beluga 70b** to create a bio and list of verbal ticks and common phrases used by each person. Then `mentors.ts` will take a question, and choose three of the 'mentors' and start a conversation with them. Occasionally, they will talk to each other, and other times they will just deliver a set of monologues. It's fun to see what they do and say.
+
+## Usage
+
+```bash
+ts-node ./character-generator.ts "Lorne Greene"
+```
+
+This will create `lornegreene/Modelfile`. Now you can create a model with this command:
+
+```bash
+ollama create lornegreene -f lornegreene/Modelfile
+```
+
+If you want to add your own mentors, you will have to update the code to look at your namespace instead of **mattw**. Also set the list of mentors to include yours.
+
+```bash
+ts-node ./mentors.ts "What is a Jackalope?"
+```

examples/typescript-mentors/character-generator.ts

@@ -0,0 +1,26 @@
+import { Ollama } from 'ollama-node'
+import fs from 'fs';
+import path from 'path';
+
+async function characterGenerator() {
+  const character = process.argv[2];
+  console.log(`You are creating a character for ${character}.`);
+  const foldername = character.replace(/\s/g, '').toLowerCase();
+  const directory = path.join(__dirname, foldername);
+  if (!fs.existsSync(directory)) {
+    fs.mkdirSync(directory, { recursive: true });
+  }
+
+  const ollama = new Ollama();
+  ollama.setModel("stablebeluga2:70b-q4_K_M");
+  const bio = await ollama.generate(`create a bio of ${character} in a single long paragraph. Instead of saying '${character} is...' or '${character} was...' use language like 'You are...' or 'You were...'. Then create a paragraph describing the speaking mannerisms and style of ${character}. Don't include anything about how ${character} looked or what they sounded like, just focus on the words they said. Instead of saying '${character} would say...' use language like 'You should say...'. If you use quotes, always use single quotes instead of double quotes. If there are any specific words or phrases you used a lot, show how you used them. `);
+
+  const thecontents = `FROM llama2\nSYSTEM """\n${bio.response.replace(/(\r\n|\n|\r)/gm, " ").replace('would', 'should')} All answers to questions should be related back to what you are most known for.\n"""`;
+
+  fs.writeFile(path.join(directory, 'Modelfile'), thecontents, (err: any) => {
+    if (err) throw err;
+    console.log('The file has been saved!');
+  });
+}
+
+characterGenerator();

examples/typescript-mentors/mentors.ts

@@ -0,0 +1,59 @@
+import { Ollama } from 'ollama-node';
+
+const mentorCount = 3;
+const ollama = new Ollama();
+
+function getMentors(): string[] {
+  const mentors = ['Gary Vaynerchuk', 'Kanye West', 'Martha Stewart', 'Neil deGrasse Tyson', 'Owen Wilson', 'Ronald Reagan', 'Donald Trump', 'Barack Obama', 'Jeff Bezos'];
+  const chosenMentors: string[] = [];
+  for (let i = 0; i < mentorCount; i++) {
+    const mentor = mentors[Math.floor(Math.random() * mentors.length)];
+    chosenMentors.push(mentor);
+    mentors.splice(mentors.indexOf(mentor), 1);
+  }
+  return chosenMentors;
+}
+
+function getMentorFileName(mentor: string): string {
+  const model = mentor.toLowerCase().replace(/\s/g, '');
+  return `mattw/${model}`;
+}
+
+async function getSystemPrompt(mentor: string, isLast: boolean, question: string): Promise<string> {
+  ollama.setModel(getMentorFileName(mentor));
+  const info = await ollama.showModelInfo()
+  let SystemPrompt = info.system || '';
+  SystemPrompt += ` You should continue the conversation as if you were ${mentor} and acknowledge the people before you in the conversation. You should adopt their mannerisms and tone, but also not use language they wouldn't use. If they are not known to know about the concept in the question, don't offer an answer. Your answer should be no longer than 1 paragraph. And definitely try not to sound like anyone else. Don't repeat any slang or phrases already used. And if it is a question the original ${mentor} wouldn't have know the answer to, just say that you don't know, in the style of ${mentor}. And think about the time the person lived. Don't use terminology that they wouldn't have used.`
+
+  if (isLast) {
+    SystemPrompt += ` End your answer with something like I hope our answers help you out`;
+  } else {
+    SystemPrompt += ` Remember, this is a conversation, so you don't need a conclusion, but end your answer with a question related to the first question: "${question}".`;
+  }
+  return SystemPrompt;
+}
+
+async function main() {
+  const mentors = getMentors();
+  const question = process.argv[2];
+  let theConversation = `Here is the conversation so far.\nYou: ${question}\n`
+
+  for await (const mentor of mentors) {
+    const SystemPrompt = await getSystemPrompt(mentor, mentor === mentors[mentorCount - 1], question);
+    ollama.setModel(getMentorFileName(mentor));
+    ollama.setSystemPrompt(SystemPrompt);
+    let output = '';
+    process.stdout.write(`\n${mentor}: `);
+    for await (const chunk of ollama.streamingGenerate(theConversation + `Continue the conversation as if you were ${mentor} on the question "${question}".`)) {
+      if (chunk.response) {
+        output += chunk.response;
+        process.stdout.write(chunk.response);
+      } else {
+        process.stdout.write('\n');
+      }
+    }
+    theConversation += `${mentor}: ${output}\n\n`
+  }
+}
+
+main();

examples/typescript-mentors/package.json

@@ -0,0 +1,7 @@
+{
+  "dependencies": {
+    "fs": "^0.0.1-security",
+    "ollama-node": "^0.0.3",
+    "path": "^0.12.7"
+  }
+}

format/bytes.go

@@ -0,0 +1,23 @@
+package format
+
+import "fmt"
+
+const (
+	Byte     = 1
+	KiloByte = Byte * 1000
+	MegaByte = KiloByte * 1000
+	GigaByte = MegaByte * 1000
+)
+
+func HumanBytes(b int64) string {
+	switch {
+	case b > GigaByte:
+		return fmt.Sprintf("%d GB", b/GigaByte)
+	case b > MegaByte:
+		return fmt.Sprintf("%d MB", b/MegaByte)
+	case b > KiloByte:
+		return fmt.Sprintf("%d KB", b/KiloByte)
+	default:
+		return fmt.Sprintf("%d B", b)
+	}
+}

format/format.go

@@ -0,0 +1,25 @@
+package format
+
+import (
+	"fmt"
+	"math"
+)
+
+const (
+	Thousand = 1000
+	Million  = Thousand * 1000
+	Billion  = Million * 1000
+)
+
+func HumanNumber(b uint64) string {
+	switch {
+	case b > Billion:
+		return fmt.Sprintf("%.0fB", math.Round(float64(b)/Billion))
+	case b > Million:
+		return fmt.Sprintf("%.0fM", math.Round(float64(b)/Million))
+	case b > Thousand:
+		return fmt.Sprintf("%.0fK", math.Round(float64(b)/Thousand))
+	default:
+		return fmt.Sprintf("%d", b)
+	}
+}

format/openssh.go

@@ -0,0 +1,102 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code originally from https://go-review.googlesource.com/c/crypto/+/218620
+
+// TODO: replace with upstream once the above change is merged and released.
+
+package format
+
+import (
+	"crypto"
+	"crypto/ed25519"
+	"crypto/rand"
+	"encoding/binary"
+	"encoding/pem"
+	"fmt"
+
+	"golang.org/x/crypto/ssh"
+)
+
+const privateKeyAuthMagic = "openssh-key-v1\x00"
+
+type openSSHEncryptedPrivateKey struct {
+	CipherName string
+	KDFName    string
+	KDFOptions string
+	KeysCount  uint32
+	PubKey     []byte
+	KeyBlocks  []byte
+}
+
+type openSSHPrivateKey struct {
+	Check1  uint32
+	Check2  uint32
+	Keytype string
+	Rest    []byte `ssh:"rest"`
+}
+
+type openSSHEd25519PrivateKey struct {
+	Pub     []byte
+	Priv    []byte
+	Comment string
+	Pad     []byte `ssh:"rest"`
+}
+
+func OpenSSHPrivateKey(key crypto.PrivateKey, comment string) (*pem.Block, error) {
+	var check uint32
+	if err := binary.Read(rand.Reader, binary.BigEndian, &check); err != nil {
+		return nil, err
+	}
+
+	var pk1 openSSHPrivateKey
+	pk1.Check1 = check
+	pk1.Check2 = check
+
+	var w openSSHEncryptedPrivateKey
+	w.KeysCount = 1
+
+	if k, ok := key.(*ed25519.PrivateKey); ok {
+		key = *k
+	}
+
+	switch k := key.(type) {
+	case ed25519.PrivateKey:
+		pub, priv := k[32:], k
+		key := openSSHEd25519PrivateKey{
+			Pub:     pub,
+			Priv:    priv,
+			Comment: comment,
+		}
+
+		pk1.Keytype = ssh.KeyAlgoED25519
+		pk1.Rest = ssh.Marshal(key)
+
+		w.PubKey = ssh.Marshal(struct {
+			KeyType string
+			Pub     []byte
+		}{
+			ssh.KeyAlgoED25519, pub,
+		})
+	default:
+		return nil, fmt.Errorf("ssh: unknown key type %T", k)
+	}
+
+	w.KeyBlocks = openSSHPadding(ssh.Marshal(pk1), 8)
+
+	w.CipherName, w.KDFName, w.KDFOptions = "none", "none", ""
+
+	return &pem.Block{
+		Type:  "OPENSSH PRIVATE KEY",
+		Bytes: append([]byte(privateKeyAuthMagic), ssh.Marshal(w)...),
+	}, nil
+}
+
+func openSSHPadding(block []byte, blocksize int) []byte {
+	for i, j := 0, len(block); (j+i)%blocksize != 0; i++ {
+		block = append(block, byte(i+1))
+	}
+
+	return block
+}