diff --git a/model/process_text_spm.go b/model/process_text_spm.go index a38d422d3..632293d83 100644 --- a/model/process_text_spm.go +++ b/model/process_text_spm.go @@ -26,7 +26,6 @@ func NewSentencePieceModel(pre string, vocab *Vocabulary) SentencePieceModel { counter := map[int]int{} var maxTokenLen int - for cnt, _ := range vocab.Types { switch vocab.Types[cnt] { case TOKEN_TYPE_NORMAL, TOKEN_TYPE_USER_DEFINED, TOKEN_TYPE_UNUSED: @@ -194,7 +193,6 @@ func (spm SentencePieceModel) Encode(s string) ([]int32, error) { } } } - } slog.Debug("encoded", "ids", ids)