jeans/ollama
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ggml-backend: Let GGML allocate context memory

Browse Source 
Passing in a Go buffer is not safe because the garbage collector could
free or move the memory while the context is still open. However, if
we pass in the size and a nil pointer then GGML will allocate it from
the C side.
This commit is contained in:
Jesse Gross 2025-01-30 17:52:19 -08:00 committed by Jesse Gross
parent d773b7d671
commit 01d9a46854
1 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
ml/backend/ggml/ggml.go
View File

@ -198,10 +198,9 @@ func (b *Backend) Get(name string) ml.Tensor {
func (b *Backend) NewContext() ml.Context { func (b *Backend) NewContext() ml.Context {
nodes := max(8192, len(b.meta.Tensors().Items())*5) nodes := max(8192, len(b.meta.Tensors().Items())*5)
bts := make([]byte, C.size_t(nodes)*C.ggml_tensor_overhead()+C.ggml_graph_overhead_custom(C.size_t(nodes), false))
c := C.ggml_init(C.struct_ggml_init_params{ c := C.ggml_init(C.struct_ggml_init_params{
mem_buffer: unsafe.Pointer(&bts[0]), mem_buffer: nil,
mem_size: C.size_t(len(bts)), mem_size: C.size_t(nodes)*C.ggml_tensor_overhead() + C.ggml_graph_overhead_custom(C.size_t(nodes), false),
no_alloc: true, no_alloc: true,
}) })