mirror of
https://github.com/ollama/ollama.git
synced 2025-05-10 18:06:33 +02:00
digest files in parallel
This commit is contained in:
parent
0f3f9e353d
commit
16fca86c4a
1 changed files with 21 additions and 5 deletions
|
@ -11,10 +11,13 @@ import (
|
|||
"os"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"golang.org/x/sync/errgroup"
|
||||
"golang.org/x/text/encoding/unicode"
|
||||
"golang.org/x/text/transform"
|
||||
|
||||
|
@ -144,12 +147,25 @@ func fileDigestMap(path string) (map[string]string, error) {
|
|||
files = []string{path}
|
||||
}
|
||||
|
||||
var mu sync.Mutex
|
||||
var g errgroup.Group
|
||||
g.SetLimit(max(runtime.GOMAXPROCS(0)-1, 1))
|
||||
for _, f := range files {
|
||||
g.Go(func() error {
|
||||
digest, err := digestForFile(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return err
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
fl[f] = digest
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
if err := g.Wait(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return fl, nil
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue