jeans/ollama
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

model: expose model vocab for structured outputs

Browse Source
This commit is contained in:
ParthSareen 2025-03-27 11:26:18 -07:00
parent 564b9b48af
commit 040e65abce
2 changed files with 6 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
model/process_text.go
View File

@ -32,6 +32,8 @@ type TextProcessor interface {
Encode(s string, addSpecial bool) ([]int32, error) Encode(s string, addSpecial bool) ([]int32, error)
Decode([]int32) (string, error) Decode([]int32) (string, error)
Is(int32, Special) bool Is(int32, Special) bool
Vocab() *Vocabulary
} }
type Vocabulary struct { type Vocabulary struct {

4
model/process_text_spm.go
View File

@ -49,6 +49,10 @@ func NewSentencePieceModel(pre string, vocab *Vocabulary) SentencePieceModel {
} }
} }
func (spm SentencePieceModel) Vocab() *Vocabulary {
return spm.vocab
}
func (spm SentencePieceModel) Is(id int32, special Special) bool { func (spm SentencePieceModel) Is(id int32, special Special) bool {
return spm.vocab.Is(id, special) return spm.vocab.Is(id, special)
} }