From 7c000ec3ed747b603bb4fe7a41fc5718acff1222 Mon Sep 17 00:00:00 2001
From: Jeremy <remy415@gmail.com>
Date: Wed, 17 Apr 2024 15:21:05 -0400
Subject: [PATCH 1/6] adds support for OLLAMA_CUSTOM_GPU_DEFS to customize GPU
 build flags

---
 llm/generate/gen_linux.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llm/generate/gen_linux.sh b/llm/generate/gen_linux.sh
index fd4a6bc09..3828fb49f 100755
--- a/llm/generate/gen_linux.sh
+++ b/llm/generate/gen_linux.sh
@@ -173,6 +173,7 @@ if [ -d "${CUDA_LIB_DIR}" ]; then
         ARM64_DEFS="-DLLAMA_AVX=off -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_CUDA_F16=off"
     fi
     CMAKE_DEFS="-DLLAMA_CUDA=on -DLLAMA_CUDA_FORCE_MMQ=on -DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES} ${COMMON_CMAKE_DEFS} ${CMAKE_DEFS} ${ARM64_DEFS}"
+    CMAKE_DEFS="${CMAKE_DEFS} ${OLLAMA_CUSTOM_GPU_DEPS}"
     BUILD_DIR="../build/linux/${ARCH}/cuda${CUDA_VARIANT}"
     EXTRA_LIBS="-L${CUDA_LIB_DIR} -lcudart -lcublas -lcublasLt -lcuda"
     build
@@ -217,6 +218,7 @@ if [ -d "${ROCM_PATH}" ]; then
     fi
     init_vars
     CMAKE_DEFS="${COMMON_CMAKE_DEFS} ${CMAKE_DEFS} -DLLAMA_HIPBLAS=on -DCMAKE_C_COMPILER=$ROCM_PATH/llvm/bin/clang -DCMAKE_CXX_COMPILER=$ROCM_PATH/llvm/bin/clang++ -DAMDGPU_TARGETS=$(amdGPUs) -DGPU_TARGETS=$(amdGPUs)"
+    CMAKE_DEFS="${CMAKE_DEFS} ${OLLAMA_CUSTOM_GPU_DEPS}"
     BUILD_DIR="../build/linux/${ARCH}/rocm${ROCM_VARIANT}"
     EXTRA_LIBS="-L${ROCM_PATH}/lib -L/opt/amdgpu/lib/x86_64-linux-gnu/ -Wl,-rpath,\$ORIGIN/../../rocm/ -lhipblas -lrocblas -lamdhip64 -lrocsolver -lamd_comgr -lhsa-runtime64 -lrocsparse -ldrm -ldrm_amdgpu"
     build

From 52f5370c484135489e994de5aee5c56cc74def37 Mon Sep 17 00:00:00 2001
From: Jeremy <remy415@gmail.com>
Date: Wed, 17 Apr 2024 16:00:48 -0400
Subject: [PATCH 2/6] add support for custom gpu build flags for llama.cpp

---
 llm/generate/gen_linux.sh | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/llm/generate/gen_linux.sh b/llm/generate/gen_linux.sh
index 3828fb49f..7368a494c 100755
--- a/llm/generate/gen_linux.sh
+++ b/llm/generate/gen_linux.sh
@@ -172,8 +172,15 @@ if [ -d "${CUDA_LIB_DIR}" ]; then
         # Disabling has minimal performance effect while maintaining compatibility. 
         ARM64_DEFS="-DLLAMA_AVX=off -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_CUDA_F16=off"
     fi
-    CMAKE_DEFS="-DLLAMA_CUDA=on -DLLAMA_CUDA_FORCE_MMQ=on -DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES} ${COMMON_CMAKE_DEFS} ${CMAKE_DEFS} ${ARM64_DEFS}"
-    CMAKE_DEFS="${CMAKE_DEFS} ${OLLAMA_CUSTOM_GPU_DEPS}"
+    # Users building from source can tune the exact flags we pass to cmake for configuring llama.cpp
+    if [ -n "${OLLAMA_CUSTOM_GPU_DEFS}" ]; then
+        echo "OLLAMA_CUSTOM_GPU_DEFS=\"${OLLAMA_CUSTOM_GPU_DEFS}\""
+        CMAKE_CUDA_DEFS="-DLLAMA_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES} ${OLLAMA_CUSTOM_GPU_DEFS}"
+        echo "Building custom GPU"
+    else
+        CMAKE_CUDA_DEFS="-DLLAMA_CUDA=on -DLLAMA_CUDA_FORCE_MMQ=on -DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}"
+    fi
+    CMAKE_DEFS="${COMMON_CMAKE_DEFS} ${CMAKE_DEFS} ${ARM64_DEFS} ${CMAKE_CUDA_DEFS}"
     BUILD_DIR="../build/linux/${ARCH}/cuda${CUDA_VARIANT}"
     EXTRA_LIBS="-L${CUDA_LIB_DIR} -lcudart -lcublas -lcublasLt -lcuda"
     build
@@ -218,7 +225,12 @@ if [ -d "${ROCM_PATH}" ]; then
     fi
     init_vars
     CMAKE_DEFS="${COMMON_CMAKE_DEFS} ${CMAKE_DEFS} -DLLAMA_HIPBLAS=on -DCMAKE_C_COMPILER=$ROCM_PATH/llvm/bin/clang -DCMAKE_CXX_COMPILER=$ROCM_PATH/llvm/bin/clang++ -DAMDGPU_TARGETS=$(amdGPUs) -DGPU_TARGETS=$(amdGPUs)"
-    CMAKE_DEFS="${CMAKE_DEFS} ${OLLAMA_CUSTOM_GPU_DEPS}"
+    # Users building from source can tune the exact flags we pass to cmake for configuring llama.cpp
+    if [ -n "${OLLAMA_CUSTOM_GPU_DEFS}" ]; then
+        echo "OLLAMA_CUSTOM_GPU_DEFS=\"${OLLAMA_CUSTOM_GPU_DEFS}\""
+        CMAKE_DEFS="${CMAKE_DEFS} ${OLLAMA_CUSTOM_GPU_DEFS}"
+        echo "Building custom GPU"
+    fi
     BUILD_DIR="../build/linux/${ARCH}/rocm${ROCM_VARIANT}"
     EXTRA_LIBS="-L${ROCM_PATH}/lib -L/opt/amdgpu/lib/x86_64-linux-gnu/ -Wl,-rpath,\$ORIGIN/../../rocm/ -lhipblas -lrocblas -lamdhip64 -lrocsolver -lamd_comgr -lhsa-runtime64 -lrocsparse -ldrm -ldrm_amdgpu"
     build

From 440b7190ed2e704290f99c1d8124a9bccd1bcc05 Mon Sep 17 00:00:00 2001
From: Jeremy <remy415@gmail.com>
Date: Thu, 18 Apr 2024 19:18:10 -0400
Subject: [PATCH 3/6] Update gen_linux.sh

Added OLLAMA_CUSTOM_CUDA_DEFS and OLLAMA_CUSTOM_ROCM_DEFS instead of OLLAMA_CUSTOM_GPU_DEFS
---
 llm/generate/gen_linux.sh | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/llm/generate/gen_linux.sh b/llm/generate/gen_linux.sh
index 7368a494c..64c204669 100755
--- a/llm/generate/gen_linux.sh
+++ b/llm/generate/gen_linux.sh
@@ -173,10 +173,10 @@ if [ -d "${CUDA_LIB_DIR}" ]; then
         ARM64_DEFS="-DLLAMA_AVX=off -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_CUDA_F16=off"
     fi
     # Users building from source can tune the exact flags we pass to cmake for configuring llama.cpp
-    if [ -n "${OLLAMA_CUSTOM_GPU_DEFS}" ]; then
-        echo "OLLAMA_CUSTOM_GPU_DEFS=\"${OLLAMA_CUSTOM_GPU_DEFS}\""
-        CMAKE_CUDA_DEFS="-DLLAMA_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES} ${OLLAMA_CUSTOM_GPU_DEFS}"
-        echo "Building custom GPU"
+    if [ -n "${OLLAMA_CUSTOM_CUDA_DEFS}" ]; then
+        echo "OLLAMA_CUSTOM_CUDA_DEFS=\"${OLLAMA_CUSTOM_CUDA_DEFS}\""
+        CMAKE_CUDA_DEFS="-DLLAMA_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES} ${OLLAMA_CUSTOM_CUDA_DEFS}"
+        echo "Building custom CUDA GPU"
     else
         CMAKE_CUDA_DEFS="-DLLAMA_CUDA=on -DLLAMA_CUDA_FORCE_MMQ=on -DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}"
     fi
@@ -226,10 +226,10 @@ if [ -d "${ROCM_PATH}" ]; then
     init_vars
     CMAKE_DEFS="${COMMON_CMAKE_DEFS} ${CMAKE_DEFS} -DLLAMA_HIPBLAS=on -DCMAKE_C_COMPILER=$ROCM_PATH/llvm/bin/clang -DCMAKE_CXX_COMPILER=$ROCM_PATH/llvm/bin/clang++ -DAMDGPU_TARGETS=$(amdGPUs) -DGPU_TARGETS=$(amdGPUs)"
     # Users building from source can tune the exact flags we pass to cmake for configuring llama.cpp
-    if [ -n "${OLLAMA_CUSTOM_GPU_DEFS}" ]; then
-        echo "OLLAMA_CUSTOM_GPU_DEFS=\"${OLLAMA_CUSTOM_GPU_DEFS}\""
-        CMAKE_DEFS="${CMAKE_DEFS} ${OLLAMA_CUSTOM_GPU_DEFS}"
-        echo "Building custom GPU"
+    if [ -n "${OLLAMA_CUSTOM_ROCM_DEFS}" ]; then
+        echo "OLLAMA_CUSTOM_ROCM_DEFS=\"${OLLAMA_CUSTOM_ROCM_DEFS}\""
+        CMAKE_DEFS="${CMAKE_DEFS} ${OLLAMA_CUSTOM_ROCM_DEFS}"
+        echo "Building custom ROCM GPU"
     fi
     BUILD_DIR="../build/linux/${ARCH}/rocm${ROCM_VARIANT}"
     EXTRA_LIBS="-L${ROCM_PATH}/lib -L/opt/amdgpu/lib/x86_64-linux-gnu/ -Wl,-rpath,\$ORIGIN/../../rocm/ -lhipblas -lrocblas -lamdhip64 -lrocsolver -lamd_comgr -lhsa-runtime64 -lrocsparse -ldrm -ldrm_amdgpu"

From 15016413de1cfeb9dbac8e8ddeec20759623bcde Mon Sep 17 00:00:00 2001
From: Jeremy <remy415@gmail.com>
Date: Thu, 18 Apr 2024 19:27:16 -0400
Subject: [PATCH 4/6] Update gen_windows.ps1

Added OLLAMA_CUSTOM_CUDA_DEFS and OLLAMA_CUSTOM_ROCM_DEFS to customize GPU builds on Windows
---
 llm/generate/gen_windows.ps1 | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/llm/generate/gen_windows.ps1 b/llm/generate/gen_windows.ps1
index 0d2ae57f1..074516d6c 100644
--- a/llm/generate/gen_windows.ps1
+++ b/llm/generate/gen_windows.ps1
@@ -243,6 +243,11 @@ if ($null -ne $script:CUDA_LIB_DIR) {
     init_vars
     $script:buildDir="../build/windows/${script:ARCH}/cuda$script:CUDA_VARIANT"
     $script:cmakeDefs += @("-A", "x64", "-DLLAMA_CUDA=ON", "-DLLAMA_AVX=on", "-DLLAMA_AVX2=off", "-DCUDAToolkit_INCLUDE_DIR=$script:CUDA_INCLUDE_DIR", "-DCMAKE_CUDA_ARCHITECTURES=${script:CMAKE_CUDA_ARCHITECTURES}")
+    if ($null -ne $script:OLLAMA_CUSTOM_CUDA_DEFS) {
+        write-host "OLLAMA_CUSTOM_CUDA_DEFS=`"${script:OLLAMA_CUSTOM_CUDA_DEFS}`"
+        $script:cmakeDefs +=@("${script:OLLAMA_CUSTOM_CUDA_DEFS}")
+        write-host "building custom CUDA GPU"
+    }
     build
     sign
     compress
@@ -274,7 +279,11 @@ if ($null -ne $env:HIP_PATH) {
 
     # We have to clobber the LIB var from the developer shell for clang to work properly
     $env:LIB=""
-
+    if ($null -ne $script:OLLAMA_CUSTOM_ROCM_DEFS) {
+        write-host "OLLAMA_CUSTOM_ROCM_DEFS=`"${script:OLLAMA_CUSTOM_ROCM_DEFS}`"
+        $script:cmakeDefs += @("${script:OLLAMA_CUSTOM_ROCM_DEFS}")
+        write-host "building custom ROCM GPU"
+    }
     write-host "Building ROCm"
     build
     # Ninja doesn't prefix with config name

From 6f18297b3a8711da2a538a0cd670e7dae48307a6 Mon Sep 17 00:00:00 2001
From: Jeremy <remy415@gmail.com>
Date: Thu, 18 Apr 2024 19:47:44 -0400
Subject: [PATCH 5/6] Update gen_windows.ps1

Forgot a " on the write-host
---
 llm/generate/gen_windows.ps1 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llm/generate/gen_windows.ps1 b/llm/generate/gen_windows.ps1
index 074516d6c..0b5576cf4 100644
--- a/llm/generate/gen_windows.ps1
+++ b/llm/generate/gen_windows.ps1
@@ -244,7 +244,7 @@ if ($null -ne $script:CUDA_LIB_DIR) {
     $script:buildDir="../build/windows/${script:ARCH}/cuda$script:CUDA_VARIANT"
     $script:cmakeDefs += @("-A", "x64", "-DLLAMA_CUDA=ON", "-DLLAMA_AVX=on", "-DLLAMA_AVX2=off", "-DCUDAToolkit_INCLUDE_DIR=$script:CUDA_INCLUDE_DIR", "-DCMAKE_CUDA_ARCHITECTURES=${script:CMAKE_CUDA_ARCHITECTURES}")
     if ($null -ne $script:OLLAMA_CUSTOM_CUDA_DEFS) {
-        write-host "OLLAMA_CUSTOM_CUDA_DEFS=`"${script:OLLAMA_CUSTOM_CUDA_DEFS}`"
+        write-host "OLLAMA_CUSTOM_CUDA_DEFS=`"${script:OLLAMA_CUSTOM_CUDA_DEFS}`""
         $script:cmakeDefs +=@("${script:OLLAMA_CUSTOM_CUDA_DEFS}")
         write-host "building custom CUDA GPU"
     }
@@ -280,7 +280,7 @@ if ($null -ne $env:HIP_PATH) {
     # We have to clobber the LIB var from the developer shell for clang to work properly
     $env:LIB=""
     if ($null -ne $script:OLLAMA_CUSTOM_ROCM_DEFS) {
-        write-host "OLLAMA_CUSTOM_ROCM_DEFS=`"${script:OLLAMA_CUSTOM_ROCM_DEFS}`"
+        write-host "OLLAMA_CUSTOM_ROCM_DEFS=`"${script:OLLAMA_CUSTOM_ROCM_DEFS}`""
         $script:cmakeDefs += @("${script:OLLAMA_CUSTOM_ROCM_DEFS}")
         write-host "building custom ROCM GPU"
     }

From 9c0db4cc83d488a9a51d262fac34aea1f651d347 Mon Sep 17 00:00:00 2001
From: Jeremy <remy415@gmail.com>
Date: Sun, 21 Apr 2024 16:13:41 -0400
Subject: [PATCH 6/6] Update gen_windows.ps1

Fixed improper env references
---
 llm/generate/gen_windows.ps1 | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/llm/generate/gen_windows.ps1 b/llm/generate/gen_windows.ps1
index 0b5576cf4..5071e09f3 100644
--- a/llm/generate/gen_windows.ps1
+++ b/llm/generate/gen_windows.ps1
@@ -243,9 +243,9 @@ if ($null -ne $script:CUDA_LIB_DIR) {
     init_vars
     $script:buildDir="../build/windows/${script:ARCH}/cuda$script:CUDA_VARIANT"
     $script:cmakeDefs += @("-A", "x64", "-DLLAMA_CUDA=ON", "-DLLAMA_AVX=on", "-DLLAMA_AVX2=off", "-DCUDAToolkit_INCLUDE_DIR=$script:CUDA_INCLUDE_DIR", "-DCMAKE_CUDA_ARCHITECTURES=${script:CMAKE_CUDA_ARCHITECTURES}")
-    if ($null -ne $script:OLLAMA_CUSTOM_CUDA_DEFS) {
-        write-host "OLLAMA_CUSTOM_CUDA_DEFS=`"${script:OLLAMA_CUSTOM_CUDA_DEFS}`""
-        $script:cmakeDefs +=@("${script:OLLAMA_CUSTOM_CUDA_DEFS}")
+    if ($null -ne $env:OLLAMA_CUSTOM_CUDA_DEFS) {
+        write-host "OLLAMA_CUSTOM_CUDA_DEFS=`"${env:OLLAMA_CUSTOM_CUDA_DEFS}`""
+        $script:cmakeDefs +=@("${env:OLLAMA_CUSTOM_CUDA_DEFS}")
         write-host "building custom CUDA GPU"
     }
     build
@@ -279,9 +279,9 @@ if ($null -ne $env:HIP_PATH) {
 
     # We have to clobber the LIB var from the developer shell for clang to work properly
     $env:LIB=""
-    if ($null -ne $script:OLLAMA_CUSTOM_ROCM_DEFS) {
-        write-host "OLLAMA_CUSTOM_ROCM_DEFS=`"${script:OLLAMA_CUSTOM_ROCM_DEFS}`""
-        $script:cmakeDefs += @("${script:OLLAMA_CUSTOM_ROCM_DEFS}")
+    if ($null -ne $env:OLLAMA_CUSTOM_ROCM_DEFS) {
+        write-host "OLLAMA_CUSTOM_ROCM_DEFS=`"${env:OLLAMA_CUSTOM_ROCM_DEFS}`""
+        $script:cmakeDefs += @("${env:OLLAMA_CUSTOM_ROCM_DEFS}")
         write-host "building custom ROCM GPU"
     }
     write-host "Building ROCm"