mirrors/ollama
1
0
Fork 0
mirror of https://github.com/ollama/ollama.git synced 2025-05-11 18:36:41 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

convert: fix parse functions

Browse source
This commit is contained in:
Michael Yang 2024-07-31 15:39:11 -07:00
parent eafc607abb
commit d8e2664c33
2 changed files with 21 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

15
convert/tokenizer.go
View file

@ -220,19 +220,22 @@ func parseVocabularyFromTokenizer(fsys fs.FS) (*Vocabulary, error) {
}
func parseVocabulary(fsys fs.FS) (*Vocabulary, error) {
patterns := map[string]func(fs.FS) (*Vocabulary, error){
"tokenizer.model": parseSentencePiece,
"tokenizer.json": parseVocabularyFromTokenizer,
patterns := []struct {
Pattern string
Func func(fs.FS) (*Vocabulary, error)
}{
{"tokenizer.model", parseSentencePiece},
{"tokenizer.json", parseVocabularyFromTokenizer},
}
for pattern, parseFn := range patterns {
if _, err := fs.Stat(fsys, pattern); errors.Is(err, os.ErrNotExist) {
for _, pattern := range patterns {
if _, err := fs.Stat(fsys, pattern.Pattern); errors.Is(err, os.ErrNotExist) {
continue
} else if err != nil {
return nil, err
}
return parseFn(fsys)
return pattern.Func(fsys)
}
return nil, errors.New("unknown tensor format")