From ba7d31240eb0b44064a8220a8a6143307b1459ef Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Mon, 3 Mar 2025 11:05:21 -0800 Subject: [PATCH] fix: own lib/ollama directory expand backend loading error handling to catch more problems and log them instead of panicing --- .../patches/0015-try-catch-backend-load.patch | 69 --------- ...-filesystem-path-instead-of-wstring.patch} | 134 ++++++++++++------ ...remove-amx.patch => 0016-remove-amx.patch} | 0 ...tch => 0017-fix-clip-compiler-error.patch} | 0 ...port.patch => 0018-add-phi4-support.patch} | 0 ml/backend/ggml/ggml/src/ggml-backend-reg.cpp | 74 +++++----- scripts/install.sh | 3 +- 7 files changed, 135 insertions(+), 145 deletions(-) delete mode 100644 llama/patches/0015-try-catch-backend-load.patch rename llama/patches/{0016-use-std-filesystem-path-instead-of-wstring.patch => 0015-use-std-filesystem-path-instead-of-wstring.patch} (67%) rename llama/patches/{0017-remove-amx.patch => 0016-remove-amx.patch} (100%) rename llama/patches/{0018-fix-clip-compiler-error.patch => 0017-fix-clip-compiler-error.patch} (100%) rename llama/patches/{0019-add-phi4-support.patch => 0018-add-phi4-support.patch} (100%) diff --git a/llama/patches/0015-try-catch-backend-load.patch b/llama/patches/0015-try-catch-backend-load.patch deleted file mode 100644 index 9aea61836..000000000 --- a/llama/patches/0015-try-catch-backend-load.patch +++ /dev/null @@ -1,69 +0,0 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: Michael Yang -Date: Tue, 11 Feb 2025 14:06:36 -0800 -Subject: [PATCH] try/catch backend load - ---- - ggml/src/ggml-backend-reg.cpp | 45 ++++++++++++++++++----------------- - 1 file changed, 23 insertions(+), 22 deletions(-) - -diff --git a/ggml/src/ggml-backend-reg.cpp b/ggml/src/ggml-backend-reg.cpp -index 98d5e14d..1c19129a 100644 ---- a/ggml/src/ggml-backend-reg.cpp -+++ b/ggml/src/ggml-backend-reg.cpp -@@ -512,32 +512,33 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent, - } - fs::directory_iterator dir_it(search_path, fs::directory_options::skip_permission_denied); - for (const auto & entry : dir_it) { -- if (entry.is_regular_file()) { -- std::wstring filename = entry.path().filename().wstring(); -- std::wstring ext = entry.path().extension().wstring(); -- if (filename.find(file_prefix) == 0 && ext == backend_filename_suffix()) { -- dl_handle_ptr handle { dl_load_library(entry.path().wstring()) }; -- if (!handle && !silent) { -- GGML_LOG_ERROR("%s: failed to load %s\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str()); -- } -- if (handle) { -+ try { -+ if (entry.is_regular_file()) { -+ std::wstring filename = entry.path().filename().wstring(); -+ std::wstring ext = entry.path().extension().wstring(); -+ if (filename.find(file_prefix) == 0 && ext == backend_filename_suffix()) { -+ dl_handle_ptr handle { dl_load_library(entry.path().wstring()) }; -+ if (!handle) { -+ GGML_LOG_ERROR("%s: failed to load %s\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str()); -+ continue; -+ } -+ - auto score_fn = (ggml_backend_score_t) dl_get_sym(handle.get(), "ggml_backend_score"); -- if (score_fn) { -- int s = score_fn(); --#ifndef NDEBUG -- GGML_LOG_DEBUG("%s: %s score: %d\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str(), s); --#endif -- if (s > best_score) { -- best_score = s; -- best_path = entry.path().wstring(); -- } -- } else { -- if (!silent) { -- GGML_LOG_INFO("%s: failed to find ggml_backend_score in %s\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str()); -- } -+ if (!score_fn) { -+ GGML_LOG_DEBUG("%s: failed to find ggml_backend_score in %s\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str()); -+ continue; -+ } -+ -+ int s = score_fn(); -+ GGML_LOG_DEBUG("%s: %s score: %d\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str(), s); -+ if (s > best_score) { -+ best_score = s; -+ best_path = entry.path().wstring(); - } - } - } -+ } catch (const std::exception & e) { -+ GGML_LOG_ERROR("%s: failed to load %s: %s\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str(), e.what()); - } - } - } diff --git a/llama/patches/0016-use-std-filesystem-path-instead-of-wstring.patch b/llama/patches/0015-use-std-filesystem-path-instead-of-wstring.patch similarity index 67% rename from llama/patches/0016-use-std-filesystem-path-instead-of-wstring.patch rename to llama/patches/0015-use-std-filesystem-path-instead-of-wstring.patch index d60066c13..e72d78ac8 100644 --- a/llama/patches/0016-use-std-filesystem-path-instead-of-wstring.patch +++ b/llama/patches/0015-use-std-filesystem-path-instead-of-wstring.patch @@ -4,11 +4,11 @@ Date: Sun, 16 Feb 2025 20:00:22 -0500 Subject: [PATCH] use std::filesystem::path instead of wstring --- - ggml/src/ggml-backend-reg.cpp | 144 ++++++++++++++-------------------- - 1 file changed, 58 insertions(+), 86 deletions(-) + ggml/src/ggml-backend-reg.cpp | 199 +++++++++++++++------------------- + 1 file changed, 88 insertions(+), 111 deletions(-) diff --git a/ggml/src/ggml-backend-reg.cpp b/ggml/src/ggml-backend-reg.cpp -index 1c19129a..c854e6bb 100644 +index 98d5e14d..799af5f3 100644 --- a/ggml/src/ggml-backend-reg.cpp +++ b/ggml/src/ggml-backend-reg.cpp @@ -66,26 +66,6 @@ @@ -264,47 +264,55 @@ index 1c19129a..c854e6bb 100644 for (const auto & search_path : search_paths) { if (!fs::exists(search_path)) { continue; -@@ -514,31 +486,31 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent, +@@ -513,29 +485,26 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent, + fs::directory_iterator dir_it(search_path, fs::directory_options::skip_permission_denied); for (const auto & entry : dir_it) { - try { - if (entry.is_regular_file()) { -- std::wstring filename = entry.path().filename().wstring(); -- std::wstring ext = entry.path().extension().wstring(); -+ std::string filename = entry.path().filename().string(); -+ std::string ext = entry.path().extension().string(); - if (filename.find(file_prefix) == 0 && ext == backend_filename_suffix()) { -- dl_handle_ptr handle { dl_load_library(entry.path().wstring()) }; -+ dl_handle_ptr handle { dl_load_library(entry.path()) }; - if (!handle) { -- GGML_LOG_ERROR("%s: failed to load %s\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str()); -+ GGML_LOG_ERROR("%s: failed to load %s\n", __func__, path_to_string(entry.path()).c_str()); - continue; - } - - auto score_fn = (ggml_backend_score_t) dl_get_sym(handle.get(), "ggml_backend_score"); - if (!score_fn) { -- GGML_LOG_DEBUG("%s: failed to find ggml_backend_score in %s\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str()); -+ GGML_LOG_DEBUG("%s: failed to find ggml_backend_score in %s\n", __func__, path_to_string(entry.path()).c_str()); - continue; - } - - int s = score_fn(); -- GGML_LOG_DEBUG("%s: %s score: %d\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str(), s); -+ GGML_LOG_DEBUG("%s: %s score: %d\n", __func__, path_to_string(entry.path()).c_str(), s); - if (s > best_score) { - best_score = s; -- best_path = entry.path().wstring(); -+ best_path = entry.path(); - } + if (entry.is_regular_file()) { +- std::wstring filename = entry.path().filename().wstring(); +- std::wstring ext = entry.path().extension().wstring(); ++ std::string filename = entry.path().filename().string(); ++ std::string ext = entry.path().extension().string(); + if (filename.find(file_prefix) == 0 && ext == backend_filename_suffix()) { +- dl_handle_ptr handle { dl_load_library(entry.path().wstring()) }; +- if (!handle && !silent) { +- GGML_LOG_ERROR("%s: failed to load %s\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str()); ++ dl_handle_ptr handle { dl_load_library(entry.path()) }; ++ if (!handle) { ++ GGML_LOG_ERROR("%s: failed to load %s\n", __func__, path_to_string(entry.path()).c_str()); ++ continue; + } +- if (handle) { +- auto score_fn = (ggml_backend_score_t) dl_get_sym(handle.get(), "ggml_backend_score"); +- if (score_fn) { +- int s = score_fn(); +-#ifndef NDEBUG +- GGML_LOG_DEBUG("%s: %s score: %d\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str(), s); +-#endif +- if (s > best_score) { +- best_score = s; +- best_path = entry.path().wstring(); +- } +- } else { +- if (!silent) { +- GGML_LOG_INFO("%s: failed to find ggml_backend_score in %s\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str()); +- } +- } ++ ++ auto score_fn = (ggml_backend_score_t) dl_get_sym(handle.get(), "ggml_backend_score"); ++ if (!score_fn) { ++ GGML_LOG_DEBUG("%s: failed to find ggml_backend_score in %s\n", __func__, path_to_string(entry.path()).c_str()); ++ continue; ++ } ++ ++ int s = score_fn(); ++ GGML_LOG_DEBUG("%s: %s score: %d\n", __func__, path_to_string(entry.path()).c_str(), s); ++ if (s > best_score) { ++ best_score = s; ++ best_path = entry.path(); } } - } catch (const std::exception & e) { -- GGML_LOG_ERROR("%s: failed to load %s: %s\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str(), e.what()); -+ GGML_LOG_ERROR("%s: failed to load %s: %s\n", __func__, path_to_string(entry.path()).c_str(), e.what()); } - } - } -@@ -546,7 +518,7 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent, +@@ -545,7 +514,7 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent, if (best_score == 0) { // try to load the base backend for (const auto & search_path : search_paths) { @@ -313,3 +321,49 @@ index 1c19129a..c854e6bb 100644 if (fs::exists(path)) { return get_reg().load_backend(path, silent); } +@@ -560,6 +529,14 @@ void ggml_backend_load_all() { + ggml_backend_load_all_from_path(nullptr); + } + ++static void ggml_backend_try_load_best(const char * name, bool silent, const char * user_search_path) { ++ try { ++ ggml_backend_load_best(name, silent, user_search_path); ++ } catch (const std::exception & e) { ++ GGML_LOG_DEBUG("%s: failed to load %s: %s\n", __func__, name, e.what()); ++ } ++} ++ + void ggml_backend_load_all_from_path(const char * dir_path) { + #ifdef NDEBUG + bool silent = true; +@@ -567,18 +544,18 @@ void ggml_backend_load_all_from_path(const char * dir_path) { + bool silent = false; + #endif + +- ggml_backend_load_best("blas", silent, dir_path); +- ggml_backend_load_best("cann", silent, dir_path); +- ggml_backend_load_best("cuda", silent, dir_path); +- ggml_backend_load_best("hip", silent, dir_path); +- ggml_backend_load_best("kompute", silent, dir_path); +- ggml_backend_load_best("metal", silent, dir_path); +- ggml_backend_load_best("rpc", silent, dir_path); +- ggml_backend_load_best("sycl", silent, dir_path); +- ggml_backend_load_best("vulkan", silent, dir_path); +- ggml_backend_load_best("opencl", silent, dir_path); +- ggml_backend_load_best("musa", silent, dir_path); +- ggml_backend_load_best("cpu", silent, dir_path); ++ ggml_backend_try_load_best("blas", silent, dir_path); ++ ggml_backend_try_load_best("cann", silent, dir_path); ++ ggml_backend_try_load_best("cuda", silent, dir_path); ++ ggml_backend_try_load_best("hip", silent, dir_path); ++ ggml_backend_try_load_best("kompute", silent, dir_path); ++ ggml_backend_try_load_best("metal", silent, dir_path); ++ ggml_backend_try_load_best("rpc", silent, dir_path); ++ ggml_backend_try_load_best("sycl", silent, dir_path); ++ ggml_backend_try_load_best("vulkan", silent, dir_path); ++ ggml_backend_try_load_best("opencl", silent, dir_path); ++ ggml_backend_try_load_best("musa", silent, dir_path); ++ ggml_backend_try_load_best("cpu", silent, dir_path); + // check the environment variable GGML_BACKEND_PATH to load an out-of-tree backend + const char * backend_path = std::getenv("GGML_BACKEND_PATH"); + if (backend_path) { diff --git a/llama/patches/0017-remove-amx.patch b/llama/patches/0016-remove-amx.patch similarity index 100% rename from llama/patches/0017-remove-amx.patch rename to llama/patches/0016-remove-amx.patch diff --git a/llama/patches/0018-fix-clip-compiler-error.patch b/llama/patches/0017-fix-clip-compiler-error.patch similarity index 100% rename from llama/patches/0018-fix-clip-compiler-error.patch rename to llama/patches/0017-fix-clip-compiler-error.patch diff --git a/llama/patches/0019-add-phi4-support.patch b/llama/patches/0018-add-phi4-support.patch similarity index 100% rename from llama/patches/0019-add-phi4-support.patch rename to llama/patches/0018-add-phi4-support.patch diff --git a/ml/backend/ggml/ggml/src/ggml-backend-reg.cpp b/ml/backend/ggml/ggml/src/ggml-backend-reg.cpp index c854e6bb2..799af5f3a 100644 --- a/ml/backend/ggml/ggml/src/ggml-backend-reg.cpp +++ b/ml/backend/ggml/ggml/src/ggml-backend-reg.cpp @@ -484,33 +484,29 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent, } fs::directory_iterator dir_it(search_path, fs::directory_options::skip_permission_denied); for (const auto & entry : dir_it) { - try { - if (entry.is_regular_file()) { - std::string filename = entry.path().filename().string(); - std::string ext = entry.path().extension().string(); - if (filename.find(file_prefix) == 0 && ext == backend_filename_suffix()) { - dl_handle_ptr handle { dl_load_library(entry.path()) }; - if (!handle) { - GGML_LOG_ERROR("%s: failed to load %s\n", __func__, path_to_string(entry.path()).c_str()); - continue; - } + if (entry.is_regular_file()) { + std::string filename = entry.path().filename().string(); + std::string ext = entry.path().extension().string(); + if (filename.find(file_prefix) == 0 && ext == backend_filename_suffix()) { + dl_handle_ptr handle { dl_load_library(entry.path()) }; + if (!handle) { + GGML_LOG_ERROR("%s: failed to load %s\n", __func__, path_to_string(entry.path()).c_str()); + continue; + } - auto score_fn = (ggml_backend_score_t) dl_get_sym(handle.get(), "ggml_backend_score"); - if (!score_fn) { - GGML_LOG_DEBUG("%s: failed to find ggml_backend_score in %s\n", __func__, path_to_string(entry.path()).c_str()); - continue; - } + auto score_fn = (ggml_backend_score_t) dl_get_sym(handle.get(), "ggml_backend_score"); + if (!score_fn) { + GGML_LOG_DEBUG("%s: failed to find ggml_backend_score in %s\n", __func__, path_to_string(entry.path()).c_str()); + continue; + } - int s = score_fn(); - GGML_LOG_DEBUG("%s: %s score: %d\n", __func__, path_to_string(entry.path()).c_str(), s); - if (s > best_score) { - best_score = s; - best_path = entry.path(); - } + int s = score_fn(); + GGML_LOG_DEBUG("%s: %s score: %d\n", __func__, path_to_string(entry.path()).c_str(), s); + if (s > best_score) { + best_score = s; + best_path = entry.path(); } } - } catch (const std::exception & e) { - GGML_LOG_ERROR("%s: failed to load %s: %s\n", __func__, path_to_string(entry.path()).c_str(), e.what()); } } } @@ -533,6 +529,14 @@ void ggml_backend_load_all() { ggml_backend_load_all_from_path(nullptr); } +static void ggml_backend_try_load_best(const char * name, bool silent, const char * user_search_path) { + try { + ggml_backend_load_best(name, silent, user_search_path); + } catch (const std::exception & e) { + GGML_LOG_DEBUG("%s: failed to load %s: %s\n", __func__, name, e.what()); + } +} + void ggml_backend_load_all_from_path(const char * dir_path) { #ifdef NDEBUG bool silent = true; @@ -540,18 +544,18 @@ void ggml_backend_load_all_from_path(const char * dir_path) { bool silent = false; #endif - ggml_backend_load_best("blas", silent, dir_path); - ggml_backend_load_best("cann", silent, dir_path); - ggml_backend_load_best("cuda", silent, dir_path); - ggml_backend_load_best("hip", silent, dir_path); - ggml_backend_load_best("kompute", silent, dir_path); - ggml_backend_load_best("metal", silent, dir_path); - ggml_backend_load_best("rpc", silent, dir_path); - ggml_backend_load_best("sycl", silent, dir_path); - ggml_backend_load_best("vulkan", silent, dir_path); - ggml_backend_load_best("opencl", silent, dir_path); - ggml_backend_load_best("musa", silent, dir_path); - ggml_backend_load_best("cpu", silent, dir_path); + ggml_backend_try_load_best("blas", silent, dir_path); + ggml_backend_try_load_best("cann", silent, dir_path); + ggml_backend_try_load_best("cuda", silent, dir_path); + ggml_backend_try_load_best("hip", silent, dir_path); + ggml_backend_try_load_best("kompute", silent, dir_path); + ggml_backend_try_load_best("metal", silent, dir_path); + ggml_backend_try_load_best("rpc", silent, dir_path); + ggml_backend_try_load_best("sycl", silent, dir_path); + ggml_backend_try_load_best("vulkan", silent, dir_path); + ggml_backend_try_load_best("opencl", silent, dir_path); + ggml_backend_try_load_best("musa", silent, dir_path); + ggml_backend_try_load_best("cpu", silent, dir_path); // check the environment variable GGML_BACKEND_PATH to load an out-of-tree backend const char * backend_path = std::getenv("GGML_BACKEND_PATH"); if (backend_path) { diff --git a/scripts/install.sh b/scripts/install.sh index 9e146e508..9c232400f 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -77,11 +77,12 @@ if [ -d "$OLLAMA_INSTALL_DIR/lib/ollama" ] ; then fi status "Installing ollama to $OLLAMA_INSTALL_DIR" $SUDO install -o0 -g0 -m755 -d $BINDIR -$SUDO install -o0 -g0 -m755 -d "$OLLAMA_INSTALL_DIR" +$SUDO install -o0 -g0 -m755 -d "$OLLAMA_INSTALL_DIR/lib/ollama" status "Downloading Linux ${ARCH} bundle" curl --fail --show-error --location --progress-bar \ "https://ollama.com/download/ollama-linux-${ARCH}.tgz${VER_PARAM}" | \ $SUDO tar -xzf - -C "$OLLAMA_INSTALL_DIR" + if [ "$OLLAMA_INSTALL_DIR/bin/ollama" != "$BINDIR/ollama" ] ; then status "Making ollama accessible in the PATH in $BINDIR" $SUDO ln -sf "$OLLAMA_INSTALL_DIR/ollama" "$BINDIR/ollama"