digest files in parallel

This commit is contained in:
Michael Yang 2025-04-04 17:33:07 -07:00 committed by Michael Yang
parent 0f3f9e353d
commit 16fca86c4a

View file

@ -11,10 +11,13 @@ import (
"os" "os"
"os/user" "os/user"
"path/filepath" "path/filepath"
"runtime"
"slices" "slices"
"strconv" "strconv"
"strings" "strings"
"sync"
"golang.org/x/sync/errgroup"
"golang.org/x/text/encoding/unicode" "golang.org/x/text/encoding/unicode"
"golang.org/x/text/transform" "golang.org/x/text/transform"
@ -144,12 +147,25 @@ func fileDigestMap(path string) (map[string]string, error) {
files = []string{path} files = []string{path}
} }
var mu sync.Mutex
var g errgroup.Group
g.SetLimit(max(runtime.GOMAXPROCS(0)-1, 1))
for _, f := range files { for _, f := range files {
digest, err := digestForFile(f) g.Go(func() error {
if err != nil { digest, err := digestForFile(f)
return nil, err if err != nil {
} return err
fl[f] = digest }
mu.Lock()
defer mu.Unlock()
fl[f] = digest
return nil
})
}
if err := g.Wait(); err != nil {
return nil, err
} }
return fl, nil return fl, nil