mirror of
https://github.com/ollama/ollama.git
synced 2025-05-12 02:46:36 +02:00
eb2b22b042
Replace large-chunk blob downloads with parallel small-chunk verification to solve timeout and performance issues. Registry users experienced progressively slowing download speeds as large-chunk transfers aged, often timing out completely. The previous approach downloaded blobs in a few large chunks but required a separate, single-threaded pass to read the entire blob back from disk for verification after download completion. This change uses the new chunksums API to fetch many smaller chunk+digest pairs, allowing concurrent downloads and immediate verification as each chunk arrives. Chunks are written directly to their final positions, eliminating the entire separate verification pass. The result is more reliable downloads that maintain speed throughout the transfer process and significantly faster overall completion, especially over unstable connections or with large blobs.
66 lines
1.6 KiB
Go
66 lines
1.6 KiB
Go
package blob
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"errors"
|
|
"io"
|
|
"os"
|
|
|
|
"github.com/ollama/ollama/server/internal/chunks"
|
|
)
|
|
|
|
type Chunk = chunks.Chunk // TODO: move chunks here?
|
|
|
|
// Chunker writes to a blob in chunks.
|
|
// Its zero value is invalid. Use [DiskCache.Chunked] to create a new Chunker.
|
|
type Chunker struct {
|
|
digest Digest
|
|
size int64
|
|
f *os.File // nil means pre-validated
|
|
}
|
|
|
|
// Chunked returns a new Chunker, ready for use storing a blob of the given
|
|
// size in chunks.
|
|
//
|
|
// Use [Chunker.Put] to write data to the blob at specific offsets.
|
|
func (c *DiskCache) Chunked(d Digest, size int64) (*Chunker, error) {
|
|
name := c.GetFile(d)
|
|
info, err := os.Stat(name)
|
|
if err == nil && info.Size() == size {
|
|
return &Chunker{}, nil
|
|
}
|
|
f, err := os.OpenFile(name, os.O_CREATE|os.O_WRONLY, 0o666)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &Chunker{digest: d, size: size, f: f}, nil
|
|
}
|
|
|
|
// Put copies chunk.Size() bytes from r to the blob at the given offset,
|
|
// merging the data with the existing blob. It returns an error if any. As a
|
|
// special case, if r has less than chunk.Size() bytes, Put returns
|
|
// io.ErrUnexpectedEOF.
|
|
func (c *Chunker) Put(chunk Chunk, d Digest, r io.Reader) error {
|
|
if c.f == nil {
|
|
return nil
|
|
}
|
|
|
|
cw := &checkWriter{
|
|
d: d,
|
|
size: chunk.Size(),
|
|
h: sha256.New(),
|
|
f: c.f,
|
|
w: io.NewOffsetWriter(c.f, chunk.Start),
|
|
}
|
|
|
|
_, err := io.CopyN(cw, r, chunk.Size())
|
|
if err != nil && errors.Is(err, io.EOF) {
|
|
return io.ErrUnexpectedEOF
|
|
}
|
|
return err
|
|
}
|
|
|
|
// Close closes the underlying file.
|
|
func (c *Chunker) Close() error {
|
|
return c.f.Close()
|
|
}
|