jeans/ollama
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

model: validate left and right pairs before merging them

Browse Source
This commit is contained in:
jmorganca 2025-03-11 18:40:55 +01:00 committed by Michael Yang
parent 63a394068c
commit 20e3593863
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
model/process_text_spm.go
View File

@ -169,6 +169,10 @@ func (spm SentencePieceModel) Encode(s string, addSpecial bool) ([]int32, error)
continue continue
} }
if id := spm.vocab.Encode(string(left.runes) + string(right.runes)); id < 0 {
continue
}
merges[pair.a].runes = append(left.runes, right.runes...) merges[pair.a].runes = append(left.runes, right.runes...)
merges[pair.b].runes = nil merges[pair.b].runes = nil
merges[pair.a].n = right.n merges[pair.a].n = right.n