docker: conditionally update hub (#2948)

docker/docker_start.sh

@@ -6,6 +6,9 @@
 set -e
 shopt -s inherit_errexit
 
+# Note that "if function_name" in bash matches when the function returns 0,
+# meaning successful execution.
+
 # match true, TRUE, True, tRuE, etc.
 istrue() {
   case "$(echo "$1" | tr '[:upper:]' '[:lower:]')" in
@@ -50,6 +53,52 @@ cscli() {
     command cscli -c "$CONFIG_FILE" "$@"
 }
 
+run_hub_update() {
+    index_modification_time=$(stat -c %Y /etc/crowdsec/hub/.index.json 2>/dev/null)
+    # Run cscli hub update if no date or if the index file is older than 24h
+    if [ -z "$index_modification_time" ] || [ $(( $(date +%s) - index_modification_time )) -gt 86400 ]; then
+        cscli hub update
+    else
+        echo "Skipping hub update, index file is recent"
+    fi
+}
+
+is_mounted() {
+    path=$(readlink -f "$1")
+    mounts=$(awk '{print $2}' /proc/mounts)
+    while true; do
+        if grep -qE ^"$path"$ <<< "$mounts"; then
+            echo "$path was found in a volume"
+            return 0
+        fi
+        path=$(dirname "$path")
+        if [ "$path" = "/" ]; then
+            return 1
+        fi
+    done
+    return 1 #unreachable
+}
+
+run_hub_update_if_from_volume() {
+    if is_mounted "/etc/crowdsec/hub/.index.json"; then
+        echo "Running hub update"
+        run_hub_update
+    else
+        echo "Skipping hub update, index file is not in a volume"
+    fi
+}
+
+run_hub_upgrade_if_from_volume() {
+    isfalse "$NO_HUB_UPGRADE" || return 0
+    if is_mounted "/var/lib/crowdsec/data"; then
+        echo "Running hub upgrade"
+        cscli hub upgrade
+    else
+        echo "Skipping hub upgrade, data directory is not in a volume"
+    fi
+
+}
+
 # conf_get <key> [file_path]
 # retrieve a value from a file (by default $CONFIG_FILE)
 conf_get() {
@@ -119,7 +168,12 @@ cscli_if_clean() {
             error_only=""
             echo "Running: cscli $error_only $itemtype $action \"$obj\" $*"
             # shellcheck disable=SC2086
-            cscli $error_only "$itemtype" "$action" "$obj" "$@"
+            if ! cscli $error_only "$itemtype" "$action" "$obj" "$@"; then
+                echo "Failed to $action $itemtype/$obj, running hub update before retrying"
+                run_hub_update
+                # shellcheck disable=SC2086
+                cscli $error_only "$itemtype" "$action" "$obj" "$@"
+            fi
         fi
     done
 }
@@ -280,9 +334,9 @@ fi
 if [ "$GID" != "" ]; then
     if istrue "$(conf_get '.db_config.type == "sqlite"')"; then
         # don't fail if the db is not there yet
-        chown -f ":$GID" "$(conf_get '.db_config.db_path')" 2>/dev/null \
-            && echo "sqlite database permissions updated" \
-            || true
+        if chown -f ":$GID" "$(conf_get '.db_config.db_path')" 2>/dev/null; then
+            echo "sqlite database permissions updated"
+        fi
     fi
 fi
 
@@ -304,11 +358,8 @@ conf_set_if "$PLUGIN_DIR" '.config_paths.plugin_dir = strenv(PLUGIN_DIR)'
 
 ## Install hub items
 
-cscli hub update || true
-
-if isfalse "$NO_HUB_UPGRADE"; then
-    cscli hub upgrade || true
-fi
+run_hub_update_if_from_volume || true
+run_hub_upgrade_if_from_volume || true
 
 cscli_if_clean parsers install crowdsecurity/docker-logs
 cscli_if_clean parsers install crowdsecurity/cri-logs

docker/preload-hub-items

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -eu
+
+# pre-download everything but don't install anything
+
+echo "Pre-downloading Hub content..."
+
+types=$(cscli hub types -o raw)
+
+for itemtype in $types; do
+    ALL_ITEMS=$(cscli "$itemtype" list -a -o json | itemtype="$itemtype" yq '.[env(itemtype)][] | .name')
+    if [[ -n "${ALL_ITEMS}" ]]; then
+        #shellcheck disable=SC2086
+        cscli "$itemtype" install \
+            $ALL_ITEMS \
+            --download-only \
+            --error
+    fi
+done
+
+echo " done."