jeans/ollama
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip: test fixes

Browse Source
This commit is contained in:
Bruce MacDonald 2025-03-20 11:19:42 -07:00 committed by jmorganca
parent edac05387f
commit 9a12fd1067
3 changed files with 1226971 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
model/process_text_test.go
View File

@ -224,12 +224,8 @@ func tekken(t testing.TB) TextProcessor {
var config struct { var config struct {
AddBosToken bool `json:"add_bos_token"` AddBosToken bool `json:"add_bos_token"`
AddEosToken bool `json:"add_eos_token"` AddEosToken bool `json:"add_eos_token"`
BosToken struct { BosToken string `json:"bos_token"`
Content string `json:"content"` EosToken string `json:"eos_token"`
} `json:"bos_token"`
EosToken struct {
Content string `json:"content"`
} `json:"eos_token"`
} }
if err := json.NewDecoder(configFile).Decode(&config); err != nil { if err := json.NewDecoder(configFile).Decode(&config); err != nil {
t.Fatal(err) t.Fatal(err)
@ -319,8 +315,8 @@ func tekken(t testing.TB) TextProcessor {
Types: types, Types: types,
Scores: scores, Scores: scores,
Merges: merges, Merges: merges,
BOS: vocab[config.BosToken.Content], BOS: vocab[config.BosToken],
EOS: vocab[config.EosToken.Content], EOS: vocab[config.EosToken],
AddBOS: config.AddBosToken, AddBOS: config.AddBosToken,
AddEOS: config.AddEosToken, AddEOS: config.AddEosToken,
} }

1217945
model/testdata/mistral-small/tokenizer.json vendored Normal file
View File

File diff suppressed because it is too large Load Diff

9020
model/testdata/mistral-small/tokenizer_config.json vendored Normal file
View File

File diff suppressed because it is too large Load Diff