model: expose model vocab for structured outputs

This commit is contained in:
ParthSareen 2025-03-27 11:26:18 -07:00
parent 564b9b48af
commit 040e65abce
2 changed files with 6 additions and 0 deletions

View File

@ -32,6 +32,8 @@ type TextProcessor interface {
Encode(s string, addSpecial bool) ([]int32, error)
Decode([]int32) (string, error)
Is(int32, Special) bool
Vocab() *Vocabulary
}
type Vocabulary struct {

View File

@ -49,6 +49,10 @@ func NewSentencePieceModel(pre string, vocab *Vocabulary) SentencePieceModel {
}
}
func (spm SentencePieceModel) Vocab() *Vocabulary {
return spm.vocab
}
func (spm SentencePieceModel) Is(id int32, special Special) bool {
return spm.vocab.Is(id, special)
}