mirror of
https://github.com/ollama/ollama.git
synced 2025-05-11 18:36:41 +02:00
eb2b22b042
Replace large-chunk blob downloads with parallel small-chunk verification to solve timeout and performance issues. Registry users experienced progressively slowing download speeds as large-chunk transfers aged, often timing out completely. The previous approach downloaded blobs in a few large chunks but required a separate, single-threaded pass to read the entire blob back from disk for verification after download completion. This change uses the new chunksums API to fetch many smaller chunk+digest pairs, allowing concurrent downloads and immediate verification as each chunk arrives. Chunks are written directly to their final positions, eliminating the entire separate verification pass. The result is more reliable downloads that maintain speed throughout the transfer process and significantly faster overall completion, especially over unstable connections or with large blobs.
81 lines
2 KiB
Go
81 lines
2 KiB
Go
package chunks
|
|
|
|
import (
|
|
"fmt"
|
|
"iter"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
type Chunk struct {
|
|
Start, End int64
|
|
}
|
|
|
|
func New(start, end int64) Chunk {
|
|
return Chunk{start, end}
|
|
}
|
|
|
|
// ParseRange parses a string in the form "unit=range" where unit is a string
|
|
// and range is a string in the form "start-end". It returns the unit and the
|
|
// range as a Chunk.
|
|
func ParseRange(s string) (unit string, _ Chunk, _ error) {
|
|
unit, r, _ := strings.Cut(s, "=")
|
|
if r == "" {
|
|
return unit, Chunk{}, nil
|
|
}
|
|
c, err := Parse(r)
|
|
if err != nil {
|
|
return "", Chunk{}, err
|
|
}
|
|
return unit, c, err
|
|
}
|
|
|
|
// Parse parses a string in the form "start-end" and returns the Chunk.
|
|
func Parse[S ~string | ~[]byte](s S) (Chunk, error) {
|
|
startPart, endPart, found := strings.Cut(string(s), "-")
|
|
if !found {
|
|
return Chunk{}, fmt.Errorf("chunks: invalid range %q: missing '-'", s)
|
|
}
|
|
start, err := strconv.ParseInt(startPart, 10, 64)
|
|
if err != nil {
|
|
return Chunk{}, fmt.Errorf("chunks: invalid start to %q: %v", s, err)
|
|
}
|
|
end, err := strconv.ParseInt(endPart, 10, 64)
|
|
if err != nil {
|
|
return Chunk{}, fmt.Errorf("chunks: invalid end to %q: %v", s, err)
|
|
}
|
|
if start > end {
|
|
return Chunk{}, fmt.Errorf("chunks: invalid range %q: start > end", s)
|
|
}
|
|
return Chunk{start, end}, nil
|
|
}
|
|
|
|
// Of returns a sequence of contiguous Chunks of size chunkSize that cover
|
|
// the range [0, size), in order.
|
|
func Of(size, chunkSize int64) iter.Seq[Chunk] {
|
|
return func(yield func(Chunk) bool) {
|
|
for start := int64(0); start < size; start += chunkSize {
|
|
end := min(start+chunkSize-1, size-1)
|
|
if !yield(Chunk{start, end}) {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Count returns the number of Chunks of size chunkSize needed to cover the
|
|
// range [0, size).
|
|
func Count(size, chunkSize int64) int64 {
|
|
return (size + chunkSize - 1) / chunkSize
|
|
}
|
|
|
|
// Size returns end minus start plus one.
|
|
func (c Chunk) Size() int64 {
|
|
return c.End - c.Start + 1
|
|
}
|
|
|
|
// String returns the string representation of the Chunk in the form
|
|
// "{start}-{end}".
|
|
func (c Chunk) String() string {
|
|
return fmt.Sprintf("%d-%d", c.Start, c.End)
|
|
}
|