server/internal/client: use chunksums for concurrent blob verification (#9746)

Replace large-chunk blob downloads with parallel small-chunk
verification to solve timeout and performance issues. Registry users
experienced progressively slowing download speeds as large-chunk
transfers aged, often timing out completely.

The previous approach downloaded blobs in a few large chunks but
required a separate, single-threaded pass to read the entire blob back
from disk for verification after download completion.

This change uses the new chunksums API to fetch many smaller
chunk+digest pairs, allowing concurrent downloads and immediate
verification as each chunk arrives. Chunks are written directly to their
final positions, eliminating the entire separate verification pass.

The result is more reliable downloads that maintain speed throughout the
transfer process and significantly faster overall completion, especially
over unstable connections or with large blobs.

server/internal/cache/blob/cache.go

@@ -146,7 +146,7 @@ func debugger(err *error) func(step string) {
 // be in either of the following forms:
 //
 //	@<digest>
-//	<name>
+//	<name>@<digest>
 //	<name>
 //
 // If a digest is provided, it is returned as is and nothing else happens.
@@ -160,8 +160,6 @@ func debugger(err *error) func(step string) {
 // hashed is passed to a PutBytes call to ensure that the manifest is in the
 // blob store. This is done to ensure that future calls to [Get] succeed in
 // these cases.
-//
-// TODO(bmizerany): Move Links/Resolve/etc. out of this package.
 func (c *DiskCache) Resolve(name string) (Digest, error) {
 	name, digest := splitNameDigest(name)
 	if digest != "" {
@@ -279,18 +277,6 @@ func (c *DiskCache) Get(d Digest) (Entry, error) {
 // It returns an error if either the name or digest is invalid, or if link
 // creation encounters any issues.
 func (c *DiskCache) Link(name string, d Digest) error {
-	// TODO(bmizerany): Move link handling from cache to registry.
-	//
-	// We originally placed links in the cache due to its storage
-	// knowledge. However, the registry likely offers better context for
-	// naming concerns, and our API design shouldn't be tightly coupled to
-	// our on-disk format.
-	//
-	// Links work effectively when independent from physical location -
-	// they can reference content with matching SHA regardless of storage
-	// location. In an upcoming change, we plan to shift this
-	// responsibility to the registry where it better aligns with the
-	// system's conceptual model.
 	manifest, err := c.manifestPath(name)
 	if err != nil {
 		return err
@@ -341,7 +327,9 @@ func (c *DiskCache) GetFile(d Digest) string {
 	return absJoin(c.dir, "blobs", filename)
 }
 
-// Links returns a sequence of links in the cache in lexical order.
+// Links returns a sequence of link names. The sequence is in lexical order.
+// Names are converted from their relative path form to their name form but are
+// not guaranteed to be valid. Callers should validate the names before using.
 func (c *DiskCache) Links() iter.Seq2[string, error] {
 	return func(yield func(string, error) bool) {
 		for path, err := range c.links() {
@@ -414,12 +402,14 @@ func (c *DiskCache) links() iter.Seq2[string, error] {
 }
 
 type checkWriter struct {
-	d    Digest
 	size int64
-	n    int64
-	h    hash.Hash
+	d    Digest
 	f    *os.File
-	err  error
+	h    hash.Hash
+
+	w   io.Writer // underlying writer; set by creator
+	n   int64
+	err error
 
 	testHookBeforeFinalWrite func(*os.File)
 }
@@ -435,6 +425,10 @@ func (w *checkWriter) seterr(err error) error {
 // underlying writer is guaranteed to be the last byte of p as verified by the
 // hash.
 func (w *checkWriter) Write(p []byte) (int, error) {
+	if w.err != nil {
+		return 0, w.err
+	}
+
 	_, err := w.h.Write(p)
 	if err != nil {
 		return 0, w.seterr(err)
@@ -453,7 +447,7 @@ func (w *checkWriter) Write(p []byte) (int, error) {
 	if nextSize > w.size {
 		return 0, w.seterr(fmt.Errorf("content exceeds expected size: %d > %d", nextSize, w.size))
 	}
-	n, err := w.f.Write(p)
+	n, err := w.w.Write(p)
 	w.n += int64(n)
 	return n, w.seterr(err)
 }
@@ -493,10 +487,12 @@ func (c *DiskCache) copyNamedFile(name string, file io.Reader, out Digest, size
 
 	// Copy file to f, but also into h to double-check hash.
 	cw := &checkWriter{
-		d:                        out,
-		size:                     size,
-		h:                        sha256.New(),
-		f:                        f,
+		d:    out,
+		size: size,
+		h:    sha256.New(),
+		f:    f,
+		w:    f,
+
 		testHookBeforeFinalWrite: c.testHookBeforeFinalWrite,
 	}
 	n, err := io.Copy(cw, file)
@@ -532,11 +528,6 @@ func splitNameDigest(s string) (name, digest string) {
 var errInvalidName = errors.New("invalid name")
 
 func nameToPath(name string) (_ string, err error) {
-	if strings.Contains(name, "@") {
-		// TODO(bmizerany): HACK: Fix names.Parse to validate.
-		// TODO(bmizerany): merge with default parts (maybe names.Merge(a, b))
-		return "", errInvalidName
-	}
 	n := names.Parse(name)
 	if !n.IsFullyQualified() {
 		return "", errInvalidName
@@ -547,8 +538,7 @@ func nameToPath(name string) (_ string, err error) {
 func absJoin(pp ...string) string {
 	abs, err := filepath.Abs(filepath.Join(pp...))
 	if err != nil {
-		// Likely a bug bug or a bad OS problem. Just panic.
-		panic(err)
+		panic(err) // this should never happen
 	}
 	return abs
 }

server/internal/cache/blob/chunked.go

@@ -0,0 +1,66 @@
+package blob
+
+import (
+	"crypto/sha256"
+	"errors"
+	"io"
+	"os"
+
+	"github.com/ollama/ollama/server/internal/chunks"
+)
+
+type Chunk = chunks.Chunk // TODO: move chunks here?
+
+// Chunker writes to a blob in chunks.
+// Its zero value is invalid. Use [DiskCache.Chunked] to create a new Chunker.
+type Chunker struct {
+	digest Digest
+	size   int64
+	f      *os.File // nil means pre-validated
+}
+
+// Chunked returns a new Chunker, ready for use storing a blob of the given
+// size in chunks.
+//
+// Use [Chunker.Put] to write data to the blob at specific offsets.
+func (c *DiskCache) Chunked(d Digest, size int64) (*Chunker, error) {
+	name := c.GetFile(d)
+	info, err := os.Stat(name)
+	if err == nil && info.Size() == size {
+		return &Chunker{}, nil
+	}
+	f, err := os.OpenFile(name, os.O_CREATE|os.O_WRONLY, 0o666)
+	if err != nil {
+		return nil, err
+	}
+	return &Chunker{digest: d, size: size, f: f}, nil
+}
+
+// Put copies chunk.Size() bytes from r to the blob at the given offset,
+// merging the data with the existing blob. It returns an error if any. As a
+// special case, if r has less than chunk.Size() bytes, Put returns
+// io.ErrUnexpectedEOF.
+func (c *Chunker) Put(chunk Chunk, d Digest, r io.Reader) error {
+	if c.f == nil {
+		return nil
+	}
+
+	cw := &checkWriter{
+		d:    d,
+		size: chunk.Size(),
+		h:    sha256.New(),
+		f:    c.f,
+		w:    io.NewOffsetWriter(c.f, chunk.Start),
+	}
+
+	_, err := io.CopyN(cw, r, chunk.Size())
+	if err != nil && errors.Is(err, io.EOF) {
+		return io.ErrUnexpectedEOF
+	}
+	return err
+}
+
+// Close closes the underlying file.
+func (c *Chunker) Close() error {
+	return c.f.Close()
+}

server/internal/cache/blob/digest.go

@@ -63,6 +63,10 @@ func (d Digest) Short() string {
 	return fmt.Sprintf("%x", d.sum[:4])
 }
 
+func (d Digest) Sum() [32]byte {
+	return d.sum
+}
+
 func (d Digest) Compare(other Digest) int {
 	return slices.Compare(d.sum[:], other.sum[:])
 }