jeans/ollama
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ml/backend/ggml: clean up

Browse Source
This commit is contained in:
Michael Yang 2025-02-28 15:42:52 -08:00
parent 2dc60d4620
commit daaf42e4a4
1 changed files with 7 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
ml/backend/ggml/ggml.go
View File

@ -84,12 +84,12 @@ func New(r *os.File, params ml.BackendParams) (ml.Backend, error) {
} }
} }
var cpuBufferTypes []*C.struct_ggml_backend_buffer_type cpuDeviceBufferType := deviceBufferType{d: C.ggml_backend_dev_by_type(C.GGML_BACKEND_DEVICE_TYPE_CPU)}
for _, d := range append(accels, append(gpus, cpus...)...) { for _, d := range append(accels, append(gpus, cpus...)...) {
switch C.ggml_backend_dev_type(d) { switch C.ggml_backend_dev_type(d) {
case C.GGML_BACKEND_DEVICE_TYPE_CPU, case C.GGML_BACKEND_DEVICE_TYPE_CPU,
C.GGML_BACKEND_DEVICE_TYPE_ACCEL: C.GGML_BACKEND_DEVICE_TYPE_ACCEL:
cpuBufferTypes = append(cpuBufferTypes, C.ggml_backend_dev_buffer_type(d)) cpuDeviceBufferType.bts = append(cpuDeviceBufferType.bts, C.ggml_backend_dev_buffer_type(d))
} }
} }
@ -98,7 +98,7 @@ func New(r *os.File, params ml.BackendParams) (ml.Backend, error) {
bt := C.ggml_backend_dev_buffer_type(d) bt := C.ggml_backend_dev_buffer_type(d)
gpuDeviceBufferTypes = append(gpuDeviceBufferTypes, deviceBufferType{ gpuDeviceBufferTypes = append(gpuDeviceBufferTypes, deviceBufferType{
d: d, d: d,
bts: append([]*C.struct_ggml_backend_buffer_type{bt}, cpuBufferTypes...), bts: append([]*C.struct_ggml_backend_buffer_type{bt}, cpuDeviceBufferType.bts...),
}) })
} }
@ -131,18 +131,17 @@ func New(r *os.File, params ml.BackendParams) (ml.Backend, error) {
splits[i] /= sum splits[i] /= sum
} }
cpuDeviceBufferTypes := deviceBufferType{C.ggml_backend_dev_by_type(C.GGML_BACKEND_DEVICE_TYPE_CPU), cpuBufferTypes} input := cpuDeviceBufferType
input := cpuDeviceBufferTypes
blocks := int(meta.KV().BlockCount()) blocks := int(meta.KV().BlockCount())
assignLayer := func(i int) (temp deviceBufferType) { assignLayer := func(i int) deviceBufferType {
if i >= params.NumGPULayers { if i >= params.NumGPULayers {
return cpuDeviceBufferTypes return cpuDeviceBufferType
} }
index := slices.IndexFunc(splits, func(f float32) bool { return float32(i)/float32(blocks+1) < f }) index := slices.IndexFunc(splits, func(f float32) bool { return float32(i)/float32(blocks+1) < f })
if index < 0 || index >= len(gpuDeviceBufferTypes) { if index < 0 || index >= len(gpuDeviceBufferTypes) {
return cpuDeviceBufferTypes return cpuDeviceBufferType
} }
return gpuDeviceBufferTypes[index] return gpuDeviceBufferTypes[index]