create blobs in parallel (#10135)

* default max term height
* error on out of tree files
This commit is contained in:
Michael Yang 2025-05-05 11:59:26 -07:00 committed by GitHub
parent 7073600797
commit d931ee8f22
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 110 additions and 22 deletions

View file

@ -139,10 +139,28 @@ func fileDigestMap(path string) (map[string]string, error) {
var files []string
if fi.IsDir() {
files, err = filesForModel(path)
fs, err := filesForModel(path)
if err != nil {
return nil, err
}
for _, f := range fs {
f, err := filepath.EvalSymlinks(f)
if err != nil {
return nil, err
}
rel, err := filepath.Rel(path, f)
if err != nil {
return nil, err
}
if !filepath.IsLocal(rel) {
return nil, fmt.Errorf("insecure path: %s", rel)
}
files = append(files, f)
}
} else {
files = []string{path}
}
@ -215,11 +233,11 @@ func filesForModel(path string) ([]string, error) {
return nil, err
}
for _, safetensor := range matches {
if ct, err := detectContentType(safetensor); err != nil {
for _, match := range matches {
if ct, err := detectContentType(match); err != nil {
return nil, err
} else if ct != contentType {
return nil, fmt.Errorf("invalid content type: expected %s for %s", ct, safetensor)
return nil, fmt.Errorf("invalid content type: expected %s for %s", ct, match)
}
}