cscli hub update: option --with-content to keep embedded items in index; use it in docker (#3192)

Dockerfile

@@ -24,8 +24,7 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 CGO_CFLAGS="-D_LARGEFILE64_
     cd crowdsec-v* && \
     ./wizard.sh --docker-mode && \
     cd - >/dev/null && \
-    cscli hub update && \
-    ./docker/preload-hub-items && \
+    cscli hub update --with-content && \
     cscli collections install crowdsecurity/linux && \
     cscli parsers install crowdsecurity/whitelists
 

Dockerfile.debian

@@ -29,8 +29,7 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
     cd crowdsec-v* && \
     ./wizard.sh --docker-mode && \
     cd - >/dev/null && \
-    cscli hub update && \
-    ./docker/preload-hub-items && \
+    cscli hub update --with-content && \
     cscli collections install crowdsecurity/linux && \
     cscli parsers install crowdsecurity/whitelists
 

cmd/crowdsec-cli/hub.go

@@ -99,9 +99,10 @@ func (cli *cliHub) newListCmd() *cobra.Command {
 	return cmd
 }
 
-func (cli *cliHub) update(ctx context.Context) error {
+func (cli *cliHub) update(ctx context.Context, withContent bool) error {
 	local := cli.cfg().Hub
 	remote := require.RemoteHub(ctx, cli.cfg())
+	remote.EmbedItemContent = withContent
 
 	// don't use require.Hub because if there is no index file, it would fail
 	hub, err := cwhub.NewHub(local, remote, log.StandardLogger())
@@ -125,6 +126,8 @@ func (cli *cliHub) update(ctx context.Context) error {
 }
 
 func (cli *cliHub) newUpdateCmd() *cobra.Command {
+	withContent := false
+
 	cmd := &cobra.Command{
 		Use:   "update",
 		Short: "Download the latest index (catalog of available configurations)",
@@ -134,10 +137,13 @@ Fetches the .index.json file from the hub, containing the list of available conf
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			return cli.update(cmd.Context())
+			return cli.update(cmd.Context(), withContent)
 		},
 	}
 
+	flags := cmd.Flags()
+	flags.BoolVar(&withContent, "with-content", false, "Download index with embedded item content")
+
 	return cmd
 }
 

docker/docker_start.sh

@@ -57,7 +57,7 @@ run_hub_update() {
     index_modification_time=$(stat -c %Y /etc/crowdsec/hub/.index.json 2>/dev/null)
     # Run cscli hub update if no date or if the index file is older than 24h
     if [ -z "$index_modification_time" ] || [ $(( $(date +%s) - index_modification_time )) -gt 86400 ]; then
-        cscli hub update
+        cscli hub update --with-content
     else
         echo "Skipping hub update, index file is recent"
     fi

pkg/cwhub/remote.go

@@ -3,6 +3,7 @@ package cwhub
 import (
 	"context"
 	"fmt"
+	"net/url"
 
 	"github.com/sirupsen/logrus"
 
@@ -11,9 +12,10 @@ import (
 
 // RemoteHubCfg is used to retrieve index and items from the remote hub.
 type RemoteHubCfg struct {
-	Branch      string
-	URLTemplate string
-	IndexPath   string
+	Branch           string
+	URLTemplate      string
+	IndexPath        string
+	EmbedItemContent bool
 }
 
 // urlTo builds the URL to download a file from the remote hub.
@@ -30,6 +32,24 @@ func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) {
 	return fmt.Sprintf(r.URLTemplate, r.Branch, remotePath), nil
 }
 
+// addURLParam adds the "with_content=true" parameter to the URL if it's not already present.
+func addURLParam(rawURL string, param string, value string) (string, error) {
+	parsedURL, err := url.Parse(rawURL)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse URL: %w", err)
+	}
+
+	query := parsedURL.Query()
+
+	if _, exists := query[param]; !exists {
+		query.Add(param, value)
+	}
+
+	parsedURL.RawQuery = query.Encode()
+
+	return parsedURL.String(), nil
+}
+
 // fetchIndex downloads the index from the hub and returns the content.
 func (r *RemoteHubCfg) fetchIndex(ctx context.Context, destPath string) (bool, error) {
 	if r == nil {
@@ -41,6 +61,13 @@ func (r *RemoteHubCfg) fetchIndex(ctx context.Context, destPath string) (bool, e
 		return false, fmt.Errorf("failed to build hub index request: %w", err)
 	}
 
+	if r.EmbedItemContent {
+		url, err = addURLParam(url, "with_content", "true")
+		if err != nil {
+			return false, fmt.Errorf("failed to add 'with_content' parameter to URL: %w", err)
+		}
+	}
+
 	downloaded, err := downloader.
 		New().
 		WithHTTPClient(hubClient).

test/lib/config/config-local

@@ -114,7 +114,7 @@ make_init_data() {
     ./instance-db config-yaml
     ./instance-db setup
 
-    "$CSCLI" --warning hub update
+    "$CSCLI" --warning hub update --with-content
 
     # preload some content and data files
     "$CSCLI" collections install crowdsecurity/linux --download-only