mirrors/ollama
1
0
Fork 0
mirror of https://github.com/ollama/ollama.git synced 2025-05-11 18:36:41 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

runner.go: Better handle return NULL values from llama.cpp

Browse source 
Llama.cpp sometimes returns NULL as a return value to report an
error. We should explicitly check for this and convert it to a Go
error rather than putting NULL in our data structures and waiting
for it to blow up later.
This commit is contained in:
Jesse Gross 2024-10-22 14:57:46 -07:00 • committed by Jesse Gross
parent 084929c293
commit de1557a0dc
3 changed files with 37 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

10
llm/server.go
View file

@ -958,7 +958,10 @@ func (s *llmServer) Tokenize(ctx context.Context, content string) ([]int, error)
if resp.StatusCode == http.StatusNotFound {
if s.model == nil {
slog.Debug("new runner detected, loading model for cgo tokenization")
m := llama.LoadModelFromFile(s.modelPath, llama.ModelParams{VocabOnly: true})
m, err := llama.LoadModelFromFile(s.modelPath, llama.ModelParams{VocabOnly: true})
if err != nil {
return nil, err
}
s.model = m
}
return s.model.Tokenize(content, false, true)
@ -1027,7 +1030,10 @@ func (s *llmServer) Detokenize(ctx context.Context, tokens []int) (string, error
if resp.StatusCode == http.StatusNotFound {
if s.model == nil {
slog.Debug("new runner detected, loading model for cgo tokenization")
m := llama.LoadModelFromFile(s.modelPath, llama.ModelParams{VocabOnly: true})
m, err := llama.LoadModelFromFile(s.modelPath, llama.ModelParams{VocabOnly: true})
if err != nil {
return "", err
}
s.model = m
}
var resp string