This commit is contained in:
Michael Yang 2025-04-25 16:15:08 -07:00 committed by Michael Yang
parent 340448d2d1
commit 54055a6dae

View file

@ -74,7 +74,6 @@ func modelHelper(t testing.TB) model.BytePairEncoding {
t.Fatal(err) t.Fatal(err)
} }
types := make([]uint32, len(vocab))
tokens := make([]string, len(vocab)) tokens := make([]string, len(vocab))
for token, id := range vocab { for token, id := range vocab {
tokens[id] = token tokens[id] = token
@ -86,7 +85,7 @@ func modelHelper(t testing.TB) model.BytePairEncoding {
``, ``,
&model.Vocabulary{ &model.Vocabulary{
Values: tokens, Values: tokens,
Types: types, Types: make([]int32, len(vocab)),
Merges: merges, Merges: merges,
}, },
) )