mirrors/ollama
1
0
Fork 0
mirror of https://github.com/ollama/ollama.git synced 2025-05-11 18:36:41 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

ollamarunner: Temporarily disable worst case graph preallocation

Browse source 
When we later have a large batch running purely on a CPU, this
results the error:
GGML_ASSERT(talloc->buffer_id >= 0)

Disabling this means that we will incrementally reallocate memory
as the graph grows.

Fixes #10410
This commit is contained in:
Jesse Gross 2025-04-29 10:48:39 -07:00 • committed by Jesse Gross
parent 6bf0b8193a
commit a27462b708
1 changed files with 6 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

10
runner/ollamarunner/runner.go
View file

@ -723,7 +723,9 @@ func (m *multiLPath) String() string {
return strings.Join(*m, ", ") return strings.Join(*m, ", ")
} }
func (s *Server) reserveWorstCaseGraph() error { // TODO(jessegross): This is causing tensor allocation failures with large batches when not offloaded
// to the GPU
/*func (s *Server) reserveWorstCaseGraph() error {
ctx := s.model.Backend().NewContext() ctx := s.model.Backend().NewContext()
defer ctx.Close() defer ctx.Close()
@ -766,7 +768,7 @@ func (s *Server) reserveWorstCaseGraph() error {
} }
return nil return nil
} }*/
func (s *Server) loadModel( func (s *Server) loadModel(
ctx context.Context, ctx context.Context,
@ -803,10 +805,10 @@ func (s *Server) loadModel(
s.seqs = make([]*Sequence, s.parallel) s.seqs = make([]*Sequence, s.parallel)
s.seqsSem = semaphore.NewWeighted(int64(s.parallel)) s.seqsSem = semaphore.NewWeighted(int64(s.parallel))
err = s.reserveWorstCaseGraph() /*err = s.reserveWorstCaseGraph()
if err != nil { if err != nil {
panic(err) panic(err)
} }*/
s.status = llm.ServerStatusReady s.status = llm.ServerStatusReady
s.ready.Done() s.ready.Done()