docker: conditionally update hub (#2948)

.github/workflows/docker-tests.yml

@@ -59,15 +59,15 @@ jobs:
           cd docker/test
           python -m pip install --upgrade pipenv wheel
 
-      #- name: "Cache virtualenvs"
+      - name: "Cache virtualenvs"
-      #  id: cache-pipenv
+        id: cache-pipenv
-      #  uses: actions/cache@v4
+        uses: actions/cache@v4
-      #  with:
+        with:
-      #    path: ~/.local/share/virtualenvs
+          path: ~/.local/share/virtualenvs
-      #    key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }}
+          key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }}
 
       - name: "Install dependencies"
-        #if: steps.cache-pipenv.outputs.cache-hit != 'true'
+        if: steps.cache-pipenv.outputs.cache-hit != 'true'
         run: |
           cd docker/test
           pipenv install --deploy

Dockerfile

@@ -25,6 +25,7 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
     ./wizard.sh --docker-mode && \
     cd - >/dev/null && \
     cscli hub update && \
+    ./docker/preload-hub-items && \
     cscli collections install crowdsecurity/linux && \
     cscli parsers install crowdsecurity/whitelists
 

Dockerfile.debian

@@ -30,6 +30,7 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
     ./wizard.sh --docker-mode && \
     cd - >/dev/null && \
     cscli hub update && \
+    ./docker/preload-hub-items && \
     cscli collections install crowdsecurity/linux && \
     cscli parsers install crowdsecurity/whitelists
 

docker/docker_start.sh

@@ -6,6 +6,9 @@
 set -e
 shopt -s inherit_errexit
 
+# Note that "if function_name" in bash matches when the function returns 0,
+# meaning successful execution.
+
 # match true, TRUE, True, tRuE, etc.
 istrue() {
   case "$(echo "$1" | tr '[:upper:]' '[:lower:]')" in
@@ -50,6 +53,52 @@ cscli() {
     command cscli -c "$CONFIG_FILE" "$@"
 }
 
+run_hub_update() {
+    index_modification_time=$(stat -c %Y /etc/crowdsec/hub/.index.json 2>/dev/null)
+    # Run cscli hub update if no date or if the index file is older than 24h
+    if [ -z "$index_modification_time" ] || [ $(( $(date +%s) - index_modification_time )) -gt 86400 ]; then
+        cscli hub update
+    else
+        echo "Skipping hub update, index file is recent"
+    fi
+}
+
+is_mounted() {
+    path=$(readlink -f "$1")
+    mounts=$(awk '{print $2}' /proc/mounts)
+    while true; do
+        if grep -qE ^"$path"$ <<< "$mounts"; then
+            echo "$path was found in a volume"
+            return 0
+        fi
+        path=$(dirname "$path")
+        if [ "$path" = "/" ]; then
+            return 1
+        fi
+    done
+    return 1 #unreachable
+}
+
+run_hub_update_if_from_volume() {
+    if is_mounted "/etc/crowdsec/hub/.index.json"; then
+        echo "Running hub update"
+        run_hub_update
+    else
+        echo "Skipping hub update, index file is not in a volume"
+    fi
+}
+
+run_hub_upgrade_if_from_volume() {
+    isfalse "$NO_HUB_UPGRADE" || return 0
+    if is_mounted "/var/lib/crowdsec/data"; then
+        echo "Running hub upgrade"
+        cscli hub upgrade
+    else
+        echo "Skipping hub upgrade, data directory is not in a volume"
+    fi
+
+}
+
 # conf_get <key> [file_path]
 # retrieve a value from a file (by default $CONFIG_FILE)
 conf_get() {
@@ -119,7 +168,12 @@ cscli_if_clean() {
             error_only=""
             echo "Running: cscli $error_only $itemtype $action \"$obj\" $*"
             # shellcheck disable=SC2086
-            cscli $error_only "$itemtype" "$action" "$obj" "$@"
+            if ! cscli $error_only "$itemtype" "$action" "$obj" "$@"; then
+                echo "Failed to $action $itemtype/$obj, running hub update before retrying"
+                run_hub_update
+                # shellcheck disable=SC2086
+                cscli $error_only "$itemtype" "$action" "$obj" "$@"
+            fi
         fi
     done
 }
@@ -280,9 +334,9 @@ fi
 if [ "$GID" != "" ]; then
     if istrue "$(conf_get '.db_config.type == "sqlite"')"; then
         # don't fail if the db is not there yet
-        chown -f ":$GID" "$(conf_get '.db_config.db_path')" 2>/dev/null \
+        if chown -f ":$GID" "$(conf_get '.db_config.db_path')" 2>/dev/null; then
-            && echo "sqlite database permissions updated" \
+            echo "sqlite database permissions updated"
-            || true
+        fi
     fi
 fi
 
@@ -304,11 +358,8 @@ conf_set_if "$PLUGIN_DIR" '.config_paths.plugin_dir = strenv(PLUGIN_DIR)'
 
 ## Install hub items
 
-cscli hub update || true
+run_hub_update_if_from_volume || true
-
+run_hub_upgrade_if_from_volume || true
-if isfalse "$NO_HUB_UPGRADE"; then
-    cscli hub upgrade || true
-fi
 
 cscli_if_clean parsers install crowdsecurity/docker-logs
 cscli_if_clean parsers install crowdsecurity/cri-logs

docker/preload-hub-items

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -eu
+
+# pre-download everything but don't install anything
+
+echo "Pre-downloading Hub content..."
+
+types=$(cscli hub types -o raw)
+
+for itemtype in $types; do
+    ALL_ITEMS=$(cscli "$itemtype" list -a -o json | itemtype="$itemtype" yq '.[env(itemtype)][] | .name')
+    if [[ -n "${ALL_ITEMS}" ]]; then
+        #shellcheck disable=SC2086
+        cscli "$itemtype" install \
+            $ALL_ITEMS \
+            --download-only \
+            --error
+    fi
+done
+
+echo " done."

test/bin/preload-hub-items

@@ -9,20 +9,12 @@ THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
 
 # pre-download everything but don't install anything
 
-echo -n "Purging existing hub..."
+echo "Pre-downloading Hub content..."
 
 types=$("$CSCLI" hub types -o raw)
 
 for itemtype in $types; do
-    "$CSCLI" "${itemtype}" delete --all --error --purge --force
+    ALL_ITEMS=$("$CSCLI" "$itemtype" list -a -o json | itemtype="$itemtype" yq '.[env(itemtype)][] | .name')
-done
-
-echo " done."
-
-echo -n "Pre-downloading Hub content..."
-
-for itemtype in $types; do
-    ALL_ITEMS=$("$CSCLI" "$itemtype" list -a -o json | jq --arg itemtype "$itemtype" -r '.[$itemtype][].name')
     if [[ -n "${ALL_ITEMS}" ]]; then
         #shellcheck disable=SC2086
         "$CSCLI" "$itemtype" install \
@@ -32,11 +24,4 @@ for itemtype in $types; do
     fi
 done
 
-# XXX: download-only works only for collections, not for parsers, scenarios, postoverflows.
+echo " done."
-# so we have to delete the links manually, and leave the downloaded files in place
-
-for itemtype in $types; do
-    "$CSCLI" "$itemtype" delete --all --error
-done
-
-echo " done."