pkg/cwhub - refact Downloader (#3382)

* pkg/cwhub - refact Downloader
* single method interfaces
* lint

cmd/crowdsec-cli/clicapi/capi.go

@@ -261,7 +261,7 @@ func (cli *cliCapi) newStatusCmd() *cobra.Command {
 		Args:              cobra.MinimumNArgs(0),
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			hub, err := require.Hub(cli.cfg(), nil, nil)
+			hub, err := require.Hub(cli.cfg(), nil)
 			if err != nil {
 				return err
 			}

cmd/crowdsec-cli/cliconsole/console.go

@@ -114,7 +114,7 @@ func (cli *cliConsole) enroll(ctx context.Context, key string, name string, over
 		}
 	}
 
-	hub, err := require.Hub(cfg, nil, nil)
+	hub, err := require.Hub(cfg, nil)
 	if err != nil {
 		return err
 	}

cmd/crowdsec-cli/clihub/hub.go

@@ -93,7 +93,7 @@ func (cli *cliHub) newListCmd() *cobra.Command {
 		Args:              cobra.NoArgs,
 		DisableAutoGenTag: true,
 		RunE: func(_ *cobra.Command, _ []string) error {
-			hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger())
+			hub, err := require.Hub(cli.cfg(), log.StandardLogger())
 			if err != nil {
 				return err
 			}
@@ -110,15 +110,15 @@ func (cli *cliHub) newListCmd() *cobra.Command {
 
 func (cli *cliHub) update(ctx context.Context, withContent bool) error {
 	local := cli.cfg().Hub
-	remote := require.RemoteHub(ctx, cli.cfg())
-
 	// don't use require.Hub because if there is no index file, it would fail
-	hub, err := cwhub.NewHub(local, remote, log.StandardLogger())
+	hub, err := cwhub.NewHub(local, log.StandardLogger())
 	if err != nil {
 		return err
 	}
 
-	if err := hub.Update(ctx, withContent); err != nil {
+	indexProvider := require.HubDownloader(ctx, cli.cfg())
+
+	if err := hub.Update(ctx, indexProvider, withContent); err != nil {
 		return fmt.Errorf("failed to update hub: %w", err)
 	}
 
@@ -166,16 +166,18 @@ cscli hub update --with-content`,
 func (cli *cliHub) upgrade(ctx context.Context, yes bool, dryRun bool, force bool) error {
 	cfg := cli.cfg()
 
-	hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger())
+	hub, err := require.Hub(cfg, log.StandardLogger())
 	if err != nil {
 		return err
 	}
 
 	plan := hubops.NewActionPlan(hub)
 
+	contentProvider := require.HubDownloader(ctx, cfg)
+
 	for _, itemType := range cwhub.ItemTypes {
 		for _, item := range hub.GetInstalledByType(itemType, true) {
-			plan.AddCommand(hubops.NewDownloadCommand(item, force))
+			plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force))
 		}
 	}
 
@@ -196,9 +198,9 @@ func (cli *cliHub) upgrade(ctx context.Context, yes bool, dryRun bool, force boo
 
 func (cli *cliHub) newUpgradeCmd() *cobra.Command {
 	var (
-		yes bool
+		yes    bool
 		dryRun bool
-		force bool
+		force  bool
 	)
 
 	cmd := &cobra.Command{

cmd/crowdsec-cli/cliitem/item.go

@@ -71,13 +71,15 @@ func (cli cliItem) NewCommand() *cobra.Command {
 func (cli cliItem) install(ctx context.Context, args []string, yes bool, dryRun bool, downloadOnly bool, force bool, ignoreError bool) error {
 	cfg := cli.cfg()
 
-	hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger())
+	hub, err := require.Hub(cfg, log.StandardLogger())
 	if err != nil {
 		return err
 	}
 
 	plan := hubops.NewActionPlan(hub)
 
+	contentProvider := require.HubDownloader(ctx, cfg)
+
 	for _, name := range args {
 		item := hub.GetItem(cli.name, name)
 		if item == nil {
@@ -91,7 +93,7 @@ func (cli cliItem) install(ctx context.Context, args []string, yes bool, dryRun
 			continue
 		}
 
-		if err = plan.AddCommand(hubops.NewDownloadCommand(item, force)); err != nil {
+		if err = plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)); err != nil {
 			return err
 		}
 
@@ -121,7 +123,7 @@ func (cli cliItem) install(ctx context.Context, args []string, yes bool, dryRun
 
 func (cli cliItem) newInstallCmd() *cobra.Command {
 	var (
-		yes bool
+		yes          bool
 		dryRun       bool
 		downloadOnly bool
 		force        bool
@@ -180,6 +182,7 @@ func (cli cliItem) removePlan(hub *cwhub.Hub, args []string, purge bool, force b
 			if err := plan.AddCommand(hubops.NewDisableCommand(item, force)); err != nil {
 				return nil, err
 			}
+
 			if purge {
 				if err := plan.AddCommand(hubops.NewPurgeCommand(item, force)); err != nil {
 					return nil, err
@@ -211,12 +214,11 @@ func (cli cliItem) removePlan(hub *cwhub.Hub, args []string, purge bool, force b
 
 		if err := plan.AddCommand(hubops.NewDisableCommand(item, force)); err != nil {
 			return nil, err
-
 		}
+
 		if purge {
 			if err := plan.AddCommand(hubops.NewPurgeCommand(item, force)); err != nil {
 				return nil, err
-
 			}
 		}
 	}
@@ -224,11 +226,10 @@ func (cli cliItem) removePlan(hub *cwhub.Hub, args []string, purge bool, force b
 	return plan, nil
 }
 
-
 func (cli cliItem) remove(ctx context.Context, args []string, yes bool, dryRun bool, purge bool, force bool, all bool) error {
 	cfg := cli.cfg()
 
-	hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger())
+	hub, err := require.Hub(cli.cfg(), log.StandardLogger())
 	if err != nil {
 		return err
 	}
@@ -253,7 +254,7 @@ func (cli cliItem) remove(ctx context.Context, args []string, yes bool, dryRun b
 
 func (cli cliItem) newRemoveCmd() *cobra.Command {
 	var (
-		yes bool
+		yes    bool
 		dryRun bool
 		purge  bool
 		force  bool
@@ -290,12 +291,12 @@ func (cli cliItem) newRemoveCmd() *cobra.Command {
 	return cmd
 }
 
-func (cli cliItem) upgradePlan(hub *cwhub.Hub, args []string, force bool, all bool) (*hubops.ActionPlan, error) {
+func (cli cliItem) upgradePlan(hub *cwhub.Hub, contentProvider cwhub.ContentProvider, args []string, force bool, all bool) (*hubops.ActionPlan, error) {
 	plan := hubops.NewActionPlan(hub)
 
 	if all {
 		for _, item := range hub.GetInstalledByType(cli.name, true) {
-			if err := plan.AddCommand(hubops.NewDownloadCommand(item, force)); err != nil {
+			if err := plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)); err != nil {
 				return nil, err
 			}
 		}
@@ -313,7 +314,7 @@ func (cli cliItem) upgradePlan(hub *cwhub.Hub, args []string, force bool, all bo
 			return nil, fmt.Errorf("can't find '%s' in %s", itemName, cli.name)
 		}
 
-		if err := plan.AddCommand(hubops.NewDownloadCommand(item, force)); err != nil {
+		if err := plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)); err != nil {
 			return nil, err
 		}
 	}
@@ -324,12 +325,14 @@ func (cli cliItem) upgradePlan(hub *cwhub.Hub, args []string, force bool, all bo
 func (cli cliItem) upgrade(ctx context.Context, args []string, yes bool, dryRun bool, force bool, all bool) error {
 	cfg := cli.cfg()
 
-	hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger())
+	hub, err := require.Hub(cfg, log.StandardLogger())
 	if err != nil {
 		return err
 	}
 
-	plan, err := cli.upgradePlan(hub, args, force, all)
+	contentProvider := require.HubDownloader(ctx, cfg)
+
+	plan, err := cli.upgradePlan(hub, contentProvider, args, force, all)
 	if err != nil {
 		return err
 	}
@@ -349,10 +352,10 @@ func (cli cliItem) upgrade(ctx context.Context, args []string, yes bool, dryRun
 
 func (cli cliItem) newUpgradeCmd() *cobra.Command {
 	var (
-		yes bool
+		yes    bool
 		dryRun bool
-		all   bool
+		all    bool
-		force bool
+		force  bool
 	)
 
 	cmd := &cobra.Command{
@@ -390,13 +393,13 @@ func (cli cliItem) inspect(ctx context.Context, args []string, url string, diff
 		cfg.Cscli.PrometheusUrl = url
 	}
 
-	remote := (*cwhub.RemoteHubCfg)(nil)
+	var contentProvider cwhub.ContentProvider
 
 	if diff {
-		remote = require.RemoteHub(ctx, cfg)
+		contentProvider = require.HubDownloader(ctx, cfg)
 	}
 
-	hub, err := require.Hub(cfg, remote, log.StandardLogger())
+	hub, err := require.Hub(cfg, log.StandardLogger())
 	if err != nil {
 		return err
 	}
@@ -408,7 +411,7 @@ func (cli cliItem) inspect(ctx context.Context, args []string, url string, diff
 		}
 
 		if diff {
-			fmt.Println(cli.whyTainted(ctx, hub, item, rev))
+			fmt.Println(cli.whyTainted(ctx, hub, contentProvider, item, rev))
 
 			continue
 		}
@@ -462,7 +465,7 @@ func (cli cliItem) newInspectCmd() *cobra.Command {
 func (cli cliItem) list(args []string, all bool) error {
 	cfg := cli.cfg()
 
-	hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger())
+	hub, err := require.Hub(cli.cfg(), log.StandardLogger())
 	if err != nil {
 		return err
 	}
@@ -498,7 +501,7 @@ func (cli cliItem) newListCmd() *cobra.Command {
 }
 
 // return the diff between the installed version and the latest version
-func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, reverse bool) (string, error) {
+func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, contentProvider cwhub.ContentProvider, reverse bool) (string, error) {
 	if !item.State.Installed {
 		return "", fmt.Errorf("'%s' is not installed", item.FQName())
 	}
@@ -509,7 +512,7 @@ func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, reverse bool)
 	}
 	defer os.Remove(dest.Name())
 
-	_, remoteURL, err := item.FetchContentTo(ctx, dest.Name())
+	_, remoteURL, err := item.FetchContentTo(ctx, contentProvider, dest.Name())
 	if err != nil {
 		return "", err
 	}
@@ -540,7 +543,7 @@ func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, reverse bool)
 	return fmt.Sprintf("%s", diff), nil
 }
 
-func (cli cliItem) whyTainted(ctx context.Context, hub *cwhub.Hub, item *cwhub.Item, reverse bool) string {
+func (cli cliItem) whyTainted(ctx context.Context, hub *cwhub.Hub, contentProvider cwhub.ContentProvider, item *cwhub.Item, reverse bool) string {
 	if !item.State.Installed {
 		return fmt.Sprintf("# %s is not installed", item.FQName())
 	}
@@ -565,7 +568,7 @@ func (cli cliItem) whyTainted(ctx context.Context, hub *cwhub.Hub, item *cwhub.I
 			ret = append(ret, err.Error())
 		}
 
-		diff, err := cli.itemDiff(ctx, sub, reverse)
+		diff, err := cli.itemDiff(ctx, sub, contentProvider, reverse)
 		if err != nil {
 			ret = append(ret, err.Error())
 		}

cmd/crowdsec-cli/cliitem/suggest.go

@@ -37,7 +37,7 @@ func suggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) str
 }
 
 func compAllItems(itemType string, args []string, toComplete string, cfg configGetter) ([]string, cobra.ShellCompDirective) {
-	hub, err := require.Hub(cfg(), nil, nil)
+	hub, err := require.Hub(cfg(), nil)
 	if err != nil {
 		return nil, cobra.ShellCompDirectiveDefault
 	}
@@ -56,7 +56,7 @@ func compAllItems(itemType string, args []string, toComplete string, cfg configG
 }
 
 func compInstalledItems(itemType string, args []string, toComplete string, cfg configGetter) ([]string, cobra.ShellCompDirective) {
-	hub, err := require.Hub(cfg(), nil, nil)
+	hub, err := require.Hub(cfg(), nil)
 	if err != nil {
 		return nil, cobra.ShellCompDirectiveDefault
 	}

cmd/crowdsec-cli/clilapi/context.go

@@ -59,7 +59,7 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
 		`,
 		DisableAutoGenTag: true,
 		RunE: func(_ *cobra.Command, _ []string) error {
-			hub, err := require.Hub(cli.cfg(), nil, nil)
+			hub, err := require.Hub(cli.cfg(), nil)
 			if err != nil {
 				return err
 			}
@@ -101,7 +101,7 @@ func (cli *cliLapi) newContextStatusCmd() *cobra.Command {
 		DisableAutoGenTag: true,
 		RunE: func(_ *cobra.Command, _ []string) error {
 			cfg := cli.cfg()
-			hub, err := require.Hub(cfg, nil, nil)
+			hub, err := require.Hub(cfg, nil)
 			if err != nil {
 				return err
 			}
@@ -153,7 +153,7 @@ cscli lapi context detect crowdsecurity/sshd-logs
 				return fmt.Errorf("failed to init expr helpers: %w", err)
 			}
 
-			hub, err := require.Hub(cfg, nil, nil)
+			hub, err := require.Hub(cfg, nil)
 			if err != nil {
 				return err
 			}

cmd/crowdsec-cli/clilapi/status.go

@@ -102,7 +102,7 @@ func (cli *cliLapi) newStatusCmd() *cobra.Command {
 		Args:              cobra.MinimumNArgs(0),
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			hub, err := require.Hub(cli.cfg(), nil, nil)
+			hub, err := require.Hub(cli.cfg(), nil)
 			if err != nil {
 				return err
 			}

cmd/crowdsec-cli/clisetup/setup.go

@@ -95,7 +95,7 @@ func (cli *cliSetup) newDetectCmd() *cobra.Command {
 
 func (cli *cliSetup) newInstallHubCmd() *cobra.Command {
 	var (
-		yes bool
+		yes    bool
 		dryRun bool
 	)
 
@@ -289,14 +289,16 @@ func (cli *cliSetup) install(ctx context.Context, yes bool, dryRun bool, fromFil
 
 	cfg := cli.cfg()
 
-	hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger())
+	hub, err := require.Hub(cfg, log.StandardLogger())
 	if err != nil {
 		return err
 	}
 
 	verbose := (cfg.Cscli.Output == "raw")
 
-	return setup.InstallHubItems(ctx, hub, input, yes, dryRun, verbose)
+	contentProvider := require.HubDownloader(ctx, cfg)
+
+	return setup.InstallHubItems(ctx, hub, contentProvider, input, yes, dryRun, verbose)
 }
 
 func (cli *cliSetup) validate(fromFile string) error {

cmd/crowdsec-cli/clisimulation/simulation.go

@@ -71,7 +71,7 @@ func (cli *cliSimulation) newEnableCmd() *cobra.Command {
 		Example:           `cscli simulation enable`,
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			hub, err := require.Hub(cli.cfg(), nil, nil)
+			hub, err := require.Hub(cli.cfg(), nil)
 			if err != nil {
 				return err
 			}

cmd/crowdsec-cli/clisupport/support.go

@@ -491,9 +491,9 @@ func (cli *cliSupport) dump(ctx context.Context, outFile string) error {
 		skipAgent = true
 	}
 
-	hub, err := require.Hub(cfg, nil, nil)
+	hub, err := require.Hub(cfg, nil)
 	if err != nil {
-		log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected")
+		log.Warn("Could not init hub, running on LAPI? Hub related information will not be collected")
 		// XXX: lapi status check requires scenarios, will return an error
 	}
 

cmd/crowdsec-cli/config_backup.go

@@ -15,7 +15,7 @@ import (
 )
 
 func (cli *cliConfig) backupHub(dirPath string) error {
-	hub, err := require.Hub(cli.cfg(), nil, nil)
+	hub, err := require.Hub(cli.cfg(), nil)
 	if err != nil {
 		return err
 	}

cmd/crowdsec-cli/config_restore.go

@@ -18,11 +18,13 @@ import (
 func (cli *cliConfig) restoreHub(ctx context.Context, dirPath string) error {
 	cfg := cli.cfg()
 
-	hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), nil)
+	hub, err := require.Hub(cfg, nil)
 	if err != nil {
 		return err
 	}
 
+	contentProvider := require.HubDownloader(ctx, cfg)
+
 	for _, itype := range cwhub.ItemTypes {
 		itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
 		if _, err = os.Stat(itemDirectory); err != nil {
@@ -53,7 +55,7 @@ func (cli *cliConfig) restoreHub(ctx context.Context, dirPath string) error {
 
 			plan := hubops.NewActionPlan(hub)
 
-			if err = plan.AddCommand(hubops.NewDownloadCommand(item, false)); err != nil {
+			if err = plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, false)); err != nil {
 				return err
 			}
 

cmd/crowdsec-cli/require/require.go

@@ -82,12 +82,11 @@ func Notifications(c *csconfig.Config) error {
 	return nil
 }
 
-// RemoteHub returns the configuration required to download hub index and items: url, branch, etc.
+func HubDownloader(ctx context.Context, c *csconfig.Config) *cwhub.Downloader {
-func RemoteHub(ctx context.Context, c *csconfig.Config) *cwhub.RemoteHubCfg {
 	// set branch in config, and log if necessary
 	branch := HubBranch(ctx, c)
 	urlTemplate := HubURLTemplate(c)
-	remote := &cwhub.RemoteHubCfg{
+	remote := &cwhub.Downloader{
 		Branch:      branch,
 		URLTemplate: urlTemplate,
 		IndexPath:   ".index.json",
@@ -98,7 +97,7 @@ func RemoteHub(ctx context.Context, c *csconfig.Config) *cwhub.RemoteHubCfg {
 
 // Hub initializes the hub. If a remote configuration is provided, it can be used to download the index and items.
 // If no remote parameter is provided, the hub can only be used for local operations.
-func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg, logger *logrus.Logger) (*cwhub.Hub, error) {
+func Hub(c *csconfig.Config, logger *logrus.Logger) (*cwhub.Hub, error) {
 	local := c.Hub
 
 	if local == nil {
@@ -110,7 +109,7 @@ func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg, logger *logrus.Logger)
 		logger.SetOutput(io.Discard)
 	}
 
-	hub, err := cwhub.NewHub(local, remote, logger)
+	hub, err := cwhub.NewHub(local, logger)
 	if err != nil {
 		return nil, err
 	}

cmd/crowdsec/serve.go

@@ -85,7 +85,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
 	}
 
 	if !cConfig.DisableAgent {
-		hub, err := cwhub.NewHub(cConfig.Hub, nil, log.StandardLogger())
+		hub, err := cwhub.NewHub(cConfig.Hub, log.StandardLogger())
 		if err != nil {
 			return nil, err
 		}
@@ -387,7 +387,7 @@ func Serve(cConfig *csconfig.Config, agentReady chan bool) error {
 	}
 
 	if !cConfig.DisableAgent {
-		hub, err := cwhub.NewHub(cConfig.Hub, nil, log.StandardLogger())
+		hub, err := cwhub.NewHub(cConfig.Hub, log.StandardLogger())
 		if err != nil {
 			return err
 		}

go.mod

@@ -29,7 +29,7 @@ require (
 	github.com/creack/pty v1.1.21 // indirect
 	github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607
 	github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26
-	github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241219154300-555e14e3988f
+	github.com/crowdsecurity/go-cs-lib v0.0.16
 	github.com/crowdsecurity/grokky v0.2.2
 	github.com/crowdsecurity/machineid v1.0.2
 	github.com/davecgh/go-spew v1.1.1

go.sum

@@ -107,10 +107,8 @@ github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607 h1:hyrYw3h
 github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607/go.mod h1:br36fEqurGYZQGit+iDYsIzW0FF6VufMbDzyyLxEuPA=
 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU=
 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
-github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241203101722-e557f9809413 h1:VIedap4s3mXM4+tM2NMm7R3E/kn79ayLZaLHDqPYVCc=
+github.com/crowdsecurity/go-cs-lib v0.0.16 h1:2/htodjwc/sfsv4deX8F/2Fzg1bOI8w3O1/BPSvvsB0=
-github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241203101722-e557f9809413/go.mod h1:XwGcvTt4lMq4Tm1IRMSKMDf0CVrnytTU8Uoofa7AR+g=
+github.com/crowdsecurity/go-cs-lib v0.0.16/go.mod h1:XwGcvTt4lMq4Tm1IRMSKMDf0CVrnytTU8Uoofa7AR+g=
-github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241219154300-555e14e3988f h1:Pd+O4UK78uQtTqbvYX+nHvqZ7TffD51uC4q0RE/podk=
-github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241219154300-555e14e3988f/go.mod h1:XwGcvTt4lMq4Tm1IRMSKMDf0CVrnytTU8Uoofa7AR+g=
 github.com/crowdsecurity/grokky v0.2.2 h1:yALsI9zqpDArYzmSSxfBq2dhYuGUTKMJq8KOEIAsuo4=
 github.com/crowdsecurity/grokky v0.2.2/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM=
 github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc=

pkg/cwhub/cwhub_test.go

@@ -57,18 +57,18 @@ func testHub(t *testing.T, update bool) *Hub {
 		os.RemoveAll(tmpDir)
 	})
 
-	remote := &RemoteHubCfg{
+	hub, err := NewHub(local, log.StandardLogger())
-		Branch:      "master",
-		URLTemplate: mockURLTemplate,
-		IndexPath:   ".index.json",
-	}
-
-	hub, err := NewHub(local, remote, log.StandardLogger())
 	require.NoError(t, err)
 
 	if update {
+		indexProvider := &Downloader{
+			Branch:      "master",
+			URLTemplate: mockURLTemplate,
+			IndexPath:   ".index.json",
+		}
+
 		ctx := context.Background()
-		err := hub.Update(ctx, false)
+		err := hub.Update(ctx, indexProvider, false)
 		require.NoError(t, err)
 	}
 

pkg/cwhub/doc.go

@@ -85,25 +85,20 @@
 //		return fmt.Errorf("collection not found")
 //	}
 //
-// To provide the remote hub configuration, use the second parameter of NewHub():
+// Some commands require an object to provide the hub index, or contents:
 //
-//	remoteHub := cwhub.RemoteHubCfg{
+//	indexProvider := cwhub.Downloader{
 //		URLTemplate: "https://cdn-hub.crowdsec.net/crowdsecurity/%s/%s",
 //		Branch: "master",
 //		IndexPath: ".index.json",
 //	}
 //
-//	hub, err := cwhub.NewHub(localHub, remoteHub, logger)
-//	if err != nil {
-//		return fmt.Errorf("unable to initialize hub: %w", err)
-//	}
-//
 // The URLTemplate is a string that will be used to build the URL of the remote hub. It must contain two
 // placeholders: the branch and the file path (it will be an index or an item).
 //
 // Before calling hub.Load(), you can update the index file by calling the Update() method:
 //
-//	err := hub.Update(context.Background())
+//	err := hub.Update(context.Background(), indexProvider)
 //	if err != nil {
 //		return fmt.Errorf("unable to update hub index: %w", err)
 //	}

pkg/cwhub/download.go

@@ -0,0 +1,124 @@
+package cwhub
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/crowdsecurity/go-cs-lib/downloader"
+)
+
+// Downloader is used to retrieve index and items from a remote hub, with cache control.
+type Downloader struct {
+	Branch      string
+	URLTemplate string
+	IndexPath   string
+}
+
+// IndexProvider retrieves and writes .index.json
+type IndexProvider interface {
+	FetchIndex(ctx context.Context, indexFile string, withContent bool, logger *logrus.Logger) (bool, error)
+}
+
+// ContentProvider retrieves and writes the YAML files with the item content.
+type ContentProvider interface {
+	FetchContent(ctx context.Context, remotePath, destPath, wantHash string, logger *logrus.Logger) (bool, string, error)
+}
+
+// urlTo builds the URL to download a file from the remote hub.
+func (d *Downloader) urlTo(remotePath string) (string, error) {
+	// the template must contain two string placeholders
+	if fmt.Sprintf(d.URLTemplate, "%s", "%s") != d.URLTemplate {
+		return "", fmt.Errorf("invalid URL template '%s'", d.URLTemplate)
+	}
+
+	return fmt.Sprintf(d.URLTemplate, d.Branch, remotePath), nil
+}
+
+// addURLParam adds a parameter with a value (ex. "with_content=true") to the URL if it's not already present.
+func addURLParam(rawURL string, param string, value string) (string, error) {
+	parsedURL, err := url.Parse(rawURL)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse URL: %w", err)
+	}
+
+	query := parsedURL.Query()
+
+	if _, exists := query[param]; !exists {
+		query.Add(param, value)
+	}
+
+	parsedURL.RawQuery = query.Encode()
+
+	return parsedURL.String(), nil
+}
+
+// FetchIndex downloads the index from the hub and writes it to the filesystem.
+// It uses a temporary file to avoid partial downloads, and won't overwrite the original
+// if it has not changed.
+func (d *Downloader) FetchIndex(ctx context.Context, destPath string, withContent bool, logger *logrus.Logger) (bool, error) {
+	url, err := d.urlTo(d.IndexPath)
+	if err != nil {
+		return false, fmt.Errorf("failed to build hub index request: %w", err)
+	}
+
+	if withContent {
+		url, err = addURLParam(url, "with_content", "true")
+		if err != nil {
+			return false, fmt.Errorf("failed to add 'with_content' parameter to URL: %w", err)
+		}
+	}
+
+	downloaded, err := downloader.
+		New().
+		WithHTTPClient(HubClient).
+		ToFile(destPath).
+		WithETagFn(downloader.SHA256).
+		CompareContent().
+		WithLogger(logger.WithField("url", url)).
+		BeforeRequest(func(_ *http.Request) {
+			fmt.Println("Downloading " + destPath)
+		}).
+		Download(ctx, url)
+	if err != nil {
+		return false, err
+	}
+
+	return downloaded, nil
+}
+
+// FetchContent downloads the content to the specified path, through a temporary file
+// to avoid partial downloads.
+// If the hash does not match, it will not overwrite and log a warning.
+func (d *Downloader) FetchContent(ctx context.Context, remotePath, destPath, wantHash string, logger *logrus.Logger) (bool, string, error) {
+	url, err := d.urlTo(remotePath)
+	if err != nil {
+		return false, "", fmt.Errorf("failed to build request: %w", err)
+	}
+
+	downloaded, err := downloader.
+		New().
+		WithHTTPClient(HubClient).
+		ToFile(destPath).
+		WithETagFn(downloader.SHA256).
+		WithMakeDirs(true).
+		WithLogger(logger.WithField("url", url)).
+		CompareContent().
+		VerifyHash("sha256", wantHash).
+		Download(ctx, url)
+
+	var hasherr downloader.HashMismatchError
+
+	switch {
+	case errors.As(err, &hasherr):
+		logger.Warnf("%s. The index file is outdated, please run 'cscli hub update' and try again", err.Error())
+	case err != nil:
+		return false, "", err
+	}
+
+	return downloaded, url, nil
+}

pkg/cwhub/download_test.go

@@ -0,0 +1,50 @@
+package cwhub
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFetchIndex(t *testing.T) {
+	ctx := context.Background()
+
+	mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Query().Get("with_content") == "true" {
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte(`Hi I'm an index with content`))
+		} else {
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte(`Hi I'm a regular index`))
+		}
+	}))
+	defer mockServer.Close()
+
+	downloader := &Downloader{
+		Branch:      "main",
+		URLTemplate: mockServer.URL + "/%s/%s",
+		IndexPath:   "index.txt",
+	}
+
+	logger := logrus.New()
+	logger.Out = io.Discard
+
+	destPath := filepath.Join(t.TempDir(), "index.txt")
+	withContent := true
+
+	downloaded, err := downloader.FetchIndex(ctx, destPath, withContent, logger)
+	require.NoError(t, err)
+	assert.True(t, downloaded)
+
+	content, err := os.ReadFile(destPath)
+	require.NoError(t, err)
+	assert.Equal(t, "Hi I'm an index with content", string(content))
+}

pkg/cwhub/errors.go

@@ -1,19 +0,0 @@
-package cwhub
-
-import (
-	"errors"
-	"fmt"
-)
-
-// ErrNilRemoteHub is returned when trying to download with a local-only configuration.
-var ErrNilRemoteHub = errors.New("remote hub configuration is not provided. Please report this issue to the developers")
-
-// IndexNotFoundError is returned when the remote hub index is not found.
-type IndexNotFoundError struct {
-	URL    string
-	Branch string
-}
-
-func (e IndexNotFoundError) Error() string {
-	return fmt.Sprintf("index not found at %s, branch '%s'. Please check the .cscli.hub_branch value if you specified it in config.yaml, or use 'master' if not sure", e.URL, e.Branch)
-}

pkg/cwhub/fetch.go

@@ -1,24 +1,15 @@
 package cwhub
 
-// Install, upgrade and remove items from the hub to the local configuration
-
 import (
 	"context"
 	"crypto"
 	"encoding/base64"
 	"encoding/hex"
-	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
-
-	"github.com/sirupsen/logrus"
-
-	"github.com/crowdsecurity/go-cs-lib/downloader"
-
 )
 
-
 // writeEmbeddedContentTo writes the embedded content to the specified path and checks the hash.
 // If the content is base64 encoded, it will be decoded before writing. Check for item.Content
 // before calling this method.
@@ -56,40 +47,9 @@ func (i *Item) writeEmbeddedContentTo(destPath, wantHash string) error {
 	return nil
 }
 
-// writeRemoteContentTo downloads the content to the specified path and checks the hash.
-func (i *Item) writeRemoteContentTo(ctx context.Context, destPath, wantHash string) (bool, string, error) {
-	url, err := i.hub.remote.urlTo(i.RemotePath)
-	if err != nil {
-		return false, "", fmt.Errorf("failed to build request: %w", err)
-	}
-
-	d := downloader.
-		New().
-		WithHTTPClient(HubClient).
-		ToFile(destPath).
-		WithETagFn(downloader.SHA256).
-		WithMakeDirs(true).
-		WithLogger(logrus.WithField("url", url)).
-		CompareContent().
-		VerifyHash("sha256", wantHash)
-
-	hasherr := downloader.HashMismatchError{}
-
-	downloaded, err := d.Download(ctx, url)
-
-	switch {
-	case errors.As(err, &hasherr):
-		i.hub.logger.Warnf("%s. The index file is outdated, please run 'cscli hub update' and try again", err.Error())
-	case err != nil:
-		return false, "", err
-	}
-
-	return downloaded, url, nil
-}
-
 // FetchContentTo writes the last version of the item's YAML file to the specified path.
 // Returns whether the file was downloaded, and the remote url for feedback purposes.
-func (i *Item) FetchContentTo(ctx context.Context, destPath string) (bool, string, error) {
+func (i *Item) FetchContentTo(ctx context.Context, contentProvider ContentProvider, destPath string) (bool, string, error) {
 	wantHash := i.latestHash()
 	if wantHash == "" {
 		return false, "", fmt.Errorf("%s: latest hash missing from index. The index file is invalid, please run 'cscli hub update' and try again", i.FQName())
@@ -104,5 +64,5 @@ func (i *Item) FetchContentTo(ctx context.Context, destPath string) (bool, strin
 		return true, fmt.Sprintf("(embedded in %s)", i.hub.local.HubIndexFile), nil
 	}
 
-	return i.writeRemoteContentTo(ctx, destPath, wantHash)
+	return contentProvider.FetchContent(ctx, i.RemotePath, destPath, wantHash, i.hub.logger)
 }

pkg/cwhub/hub.go

@@ -22,7 +22,6 @@ type Hub struct {
 	items     HubItems // Items read from HubDir and InstallDir
 	pathIndex map[string]*Item
 	local     *csconfig.LocalHubCfg
-	remote    *RemoteHubCfg
 	logger    *logrus.Logger
 	Warnings  []string // Warnings encountered during sync
 }
@@ -35,8 +34,7 @@ func (h *Hub) GetDataDir() string {
 // NewHub returns a new Hub instance with local and (optionally) remote configuration.
 // The hub is not synced automatically. Load() must be called to read the index, sync the local state,
 // and check for unmanaged items.
-// All download operations (including updateIndex) return ErrNilRemoteHub if the remote configuration is not set.
+func NewHub(local *csconfig.LocalHubCfg, logger *logrus.Logger) (*Hub, error) {
-func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, logger *logrus.Logger) (*Hub, error) {
 	if local == nil {
 		return nil, errors.New("no hub configuration found")
 	}
@@ -48,7 +46,6 @@ func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, logger *logrus.Lo
 
 	hub := &Hub{
 		local:     local,
-		remote:    remote,
 		logger:    logger,
 		pathIndex: make(map[string]*Item, 0),
 	}
@@ -158,13 +155,13 @@ func (h *Hub) ItemStats() []string {
 
 // Update downloads the latest version of the index and writes it to disk if it changed.
 // It cannot be called after Load() unless the hub is completely empty.
-func (h *Hub) Update(ctx context.Context, withContent bool) error {
+func (h *Hub) Update(ctx context.Context, indexProvider IndexProvider, withContent bool) error {
 	if len(h.pathIndex) > 0 {
 		// if this happens, it's a bug.
 		return errors.New("cannot update hub after items have been loaded")
 	}
 
-	downloaded, err := h.remote.fetchIndex(ctx, h.local.HubIndexFile, withContent)
+	downloaded, err := indexProvider.FetchIndex(ctx, h.local.HubIndexFile, withContent, h.logger)
 	if err != nil {
 		return err
 	}
@@ -238,6 +235,7 @@ func (h *Hub) GetItemsByType(itemType string, sorted bool) []*Item {
 	}
 
 	idx := 0
+
 	for _, item := range items {
 		ret[idx] = item
 		idx += 1
@@ -269,6 +267,7 @@ func (h *Hub) GetInstalledListForAPI() []string {
 	ret := make([]string, len(scenarios)+len(appsecRules))
 
 	idx := 0
+
 	for _, item := range scenarios {
 		ret[idx] = item.Name
 		idx += 1

pkg/cwhub/hub_test.go

@@ -13,18 +13,19 @@ import (
 
 func TestInitHubUpdate(t *testing.T) {
 	hub := envSetup(t)
-	remote := &RemoteHubCfg{
+
+	_, err := NewHub(hub.local, nil)
+	require.NoError(t, err)
+
+	ctx := context.Background()
+
+	indexProvider := &Downloader{
 		URLTemplate: mockURLTemplate,
 		Branch:      "master",
 		IndexPath:   ".index.json",
 	}
 
-	_, err := NewHub(hub.local, remote, nil)
+	err = hub.Update(ctx, indexProvider, false)
-	require.NoError(t, err)
-
-	ctx := context.Background()
-
-	err = hub.Update(ctx, false)
 	require.NoError(t, err)
 
 	err = hub.Load()
@@ -48,29 +49,29 @@ func TestUpdateIndex(t *testing.T) {
 
 	hub := envSetup(t)
 
-	hub.remote = &RemoteHubCfg{
+	hub.local.HubIndexFile = tmpIndex.Name()
+
+	ctx := context.Background()
+
+	indexProvider := &Downloader{
 		URLTemplate: "x",
 		Branch:      "",
 		IndexPath:   "",
 	}
 
-	hub.local.HubIndexFile = tmpIndex.Name()
+	err = hub.Update(ctx, indexProvider, false)
-
-	ctx := context.Background()
-
-	err = hub.Update(ctx, false)
 	cstest.RequireErrorContains(t, err, "failed to build hub index request: invalid URL template 'x'")
 
 	// bad domain
 	fmt.Println("Test 'bad domain'")
 
-	hub.remote = &RemoteHubCfg{
+	indexProvider = &Downloader{
 		URLTemplate: "https://baddomain/crowdsecurity/%s/%s",
 		Branch:      "master",
 		IndexPath:   ".index.json",
 	}
 
-	err = hub.Update(ctx, false)
+	err = hub.Update(ctx, indexProvider, false)
 	require.NoError(t, err)
 	// XXX: this is not failing
 	//	cstest.RequireErrorContains(t, err, "failed http request for hub index: Get")
@@ -78,7 +79,7 @@ func TestUpdateIndex(t *testing.T) {
 	// bad target path
 	fmt.Println("Test 'bad target path'")
 
-	hub.remote = &RemoteHubCfg{
+	indexProvider = &Downloader{
 		URLTemplate: mockURLTemplate,
 		Branch:      "master",
 		IndexPath:   ".index.json",
@@ -86,6 +87,6 @@ func TestUpdateIndex(t *testing.T) {
 
 	hub.local.HubIndexFile = "/does/not/exist/index.json"
 
-	err = hub.Update(ctx, false)
+	err = hub.Update(ctx, indexProvider, false)
 	cstest.RequireErrorContains(t, err, "failed to create temporary download file for /does/not/exist/index.json:")
 }

pkg/cwhub/itemupgrade_test.go

@@ -38,7 +38,7 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
 	// collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario"
 	pushUpdateToCollectionInHub()
 
-	remote := &RemoteHubCfg{
+	remote := &Downloader{
 		URLTemplate: mockURLTemplate,
 		Branch:      "master",
 		IndexPath:   ".index.json",
@@ -98,7 +98,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
 	require.NoError(t, err)
 	require.True(t, didRemove)
 
-	remote := &RemoteHubCfg{
+	remote := &Downloader{
 		URLTemplate: mockURLTemplate,
 		Branch:      "master",
 		IndexPath:   ".index.json",
@@ -132,7 +132,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
 }
 
 // getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test.
-func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCfg) *Hub {
+func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *Downloader) *Hub {
 	hub, err := NewHub(local, remote, nil)
 	require.NoError(t, err)
 
@@ -170,7 +170,7 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te
 	require.NoError(t, err)
 	require.True(t, didRemove)
 
-	remote := &RemoteHubCfg{
+	remote := &Downloader{
 		URLTemplate: mockURLTemplate,
 		Branch:      "master",
 		IndexPath:   ".index.json",

pkg/cwhub/remote.go

@@ -1,87 +0,0 @@
-package cwhub
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"net/url"
-
-	"github.com/sirupsen/logrus"
-
-	"github.com/crowdsecurity/go-cs-lib/downloader"
-)
-
-// RemoteHubCfg is used to retrieve index and items from the remote hub.
-type RemoteHubCfg struct {
-	Branch           string
-	URLTemplate      string
-	IndexPath        string
-}
-
-// urlTo builds the URL to download a file from the remote hub.
-func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) {
-	if r == nil {
-		return "", ErrNilRemoteHub
-	}
-
-	// the template must contain two string placeholders
-	if fmt.Sprintf(r.URLTemplate, "%s", "%s") != r.URLTemplate {
-		return "", fmt.Errorf("invalid URL template '%s'", r.URLTemplate)
-	}
-
-	return fmt.Sprintf(r.URLTemplate, r.Branch, remotePath), nil
-}
-
-// addURLParam adds a parameter with a value (ex. "with_content=true") to the URL if it's not already present.
-func addURLParam(rawURL string, param string, value string) (string, error) {
-	parsedURL, err := url.Parse(rawURL)
-	if err != nil {
-		return "", fmt.Errorf("failed to parse URL: %w", err)
-	}
-
-	query := parsedURL.Query()
-
-	if _, exists := query[param]; !exists {
-		query.Add(param, value)
-	}
-
-	parsedURL.RawQuery = query.Encode()
-
-	return parsedURL.String(), nil
-}
-
-// fetchIndex downloads the index from the hub and returns the content.
-func (r *RemoteHubCfg) fetchIndex(ctx context.Context, destPath string, withContent bool) (bool, error) {
-	if r == nil {
-		return false, ErrNilRemoteHub
-	}
-
-	url, err := r.urlTo(r.IndexPath)
-	if err != nil {
-		return false, fmt.Errorf("failed to build hub index request: %w", err)
-	}
-
-	if withContent {
-		url, err = addURLParam(url, "with_content", "true")
-		if err != nil {
-			return false, fmt.Errorf("failed to add 'with_content' parameter to URL: %w", err)
-		}
-	}
-
-	downloaded, err := downloader.
-		New().
-		WithHTTPClient(HubClient).
-		ToFile(destPath).
-		WithETagFn(downloader.SHA256).
-		CompareContent().
-		WithLogger(logrus.WithField("url", url)).
-		BeforeRequest(func(_ *http.Request) {
-			fmt.Println("Downloading "+destPath)
-		}).
-		Download(ctx, url)
-	if err != nil {
-		return false, err
-	}
-
-	return downloaded, nil
-}

pkg/hubops/download.go

@@ -9,9 +9,9 @@ import (
 	"os"
 	"time"
 
+	"github.com/fatih/color"
 	"github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v3"
-	"github.com/fatih/color"
 
 	"github.com/crowdsecurity/go-cs-lib/downloader"
 
@@ -19,19 +19,19 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
-
 // DownloadCommand handles the downloading of hub items.
 // It ensures that items are fetched from the hub (or from the index file if it also has content)
 // managing dependencies and verifying the integrity of downloaded content.
 // This is used by "cscli install" and "cscli upgrade".
 // Tainted items require the force parameter, local items are skipped.
 type DownloadCommand struct {
-	Item  *cwhub.Item
+	Item            *cwhub.Item
-	Force bool
+	Force           bool
+	contentProvider cwhub.ContentProvider
 }
 
-func NewDownloadCommand(item *cwhub.Item, force bool) *DownloadCommand {
+func NewDownloadCommand(item *cwhub.Item, contentProvider cwhub.ContentProvider, force bool) *DownloadCommand {
-	return &DownloadCommand{Item: item, Force: force}
+	return &DownloadCommand{Item: item, Force: force, contentProvider: contentProvider}
 }
 
 func (c *DownloadCommand) Prepare(plan *ActionPlan) (bool, error) {
@@ -60,7 +60,7 @@ func (c *DownloadCommand) Prepare(plan *ActionPlan) (bool, error) {
 	}
 
 	for sub := range i.LatestDependencies().SubItems(plan.hub) {
-		if err := plan.AddCommand(NewDownloadCommand(sub, c.Force)); err != nil {
+		if err := plan.AddCommand(NewDownloadCommand(sub, c.contentProvider, c.Force)); err != nil {
 			return false, err
 		}
 
@@ -158,7 +158,7 @@ func (c *DownloadCommand) Run(ctx context.Context, plan *ActionPlan) error {
 		return err
 	}
 
-	downloaded, _, err := i.FetchContentTo(ctx, finalPath)
+	downloaded, _, err := i.FetchContentTo(ctx, c.contentProvider, finalPath)
 	if err != nil {
 		return fmt.Errorf("%s: %w", i.FQName(), err)
 	}

pkg/hubtest/hubtest.go

@@ -14,8 +14,8 @@ type HubTest struct {
 	CrowdSecPath              string
 	CscliPath                 string
 	HubPath                   string
-	HubTestPath               string //generic parser/scenario tests .tests
+	HubTestPath               string // generic parser/scenario tests .tests
-	HubAppsecTestPath         string //dir specific to appsec tests .appsec-tests
+	HubAppsecTestPath         string // dir specific to appsec tests .appsec-tests
 	HubIndexFile              string
 	TemplateConfigPath        string
 	TemplateProfilePath       string
@@ -93,7 +93,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecT
 			InstallDataDir: HubTestPath,
 		}
 
-		hub, err := cwhub.NewHub(local, nil, nil)
+		hub, err := cwhub.NewHub(local, nil)
 		if err != nil {
 			return HubTest{}, err
 		}
@@ -130,7 +130,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecT
 		InstallDataDir: HubTestPath,
 	}
 
-	hub, err := cwhub.NewHub(local, nil, nil)
+	hub, err := cwhub.NewHub(local, nil)
 	if err != nil {
 		return HubTest{}, err
 	}

pkg/hubtest/hubtest_item.go

@@ -212,7 +212,7 @@ func (t *HubTestItem) InstallHub() error {
 	}
 
 	// load installed hub
-	hub, err := cwhub.NewHub(t.RuntimeHubConfig, nil, nil)
+	hub, err := cwhub.NewHub(t.RuntimeHubConfig, nil)
 	if err != nil {
 		return err
 	}

pkg/leakybucket/buckets_test.go

@@ -46,7 +46,7 @@ func TestBucket(t *testing.T) {
 		InstallDataDir: testdata,
 	}
 
-	hub, err := cwhub.NewHub(hubCfg, nil, nil)
+	hub, err := cwhub.NewHub(hubCfg, nil)
 	require.NoError(t, err)
 
 	err = hub.Load()
@@ -140,6 +140,7 @@ func testOneBucket(t *testing.T, hub *cwhub.Hub, dir string, tomb *tomb.Tomb) er
 	}
 
 	scenarios := []*cwhub.Item{}
+
 	for _, x := range stages {
 		// XXX: LoadBuckets should take an interface, BucketProvider ScenarioProvider or w/e
 		item := &cwhub.Item{

pkg/setup/install.go

@@ -48,7 +48,7 @@ func decodeSetup(input []byte, fancyErrors bool) (Setup, error) {
 }
 
 // InstallHubItems installs the objects recommended in a setup file.
-func InstallHubItems(ctx context.Context, hub *cwhub.Hub, input []byte, yes, dryRun, verbose bool) error {
+func InstallHubItems(ctx context.Context, hub *cwhub.Hub, contentProvider cwhub.ContentProvider, input []byte, yes, dryRun, verbose bool) error {
 	setupEnvelope, err := decodeSetup(input, false)
 	if err != nil {
 		return err
@@ -71,7 +71,8 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, input []byte, yes, dry
 				return fmt.Errorf("collection %s not found", collection)
 			}
 
-			plan.AddCommand(hubops.NewDownloadCommand(item, forceAction))
+			plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, forceAction))
+
 			if !downloadOnly {
 				plan.AddCommand(hubops.NewEnableCommand(item, forceAction))
 			}
@@ -83,7 +84,8 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, input []byte, yes, dry
 				return fmt.Errorf("parser %s not found", parser)
 			}
 
-			plan.AddCommand(hubops.NewDownloadCommand(item, forceAction))
+			plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, forceAction))
+
 			if !downloadOnly {
 				plan.AddCommand(hubops.NewEnableCommand(item, forceAction))
 			}
@@ -95,7 +97,8 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, input []byte, yes, dry
 				return fmt.Errorf("scenario %s not found", scenario)
 			}
 
-			plan.AddCommand(hubops.NewDownloadCommand(item, forceAction))
+			plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, forceAction))
+
 			if !downloadOnly {
 				plan.AddCommand(hubops.NewEnableCommand(item, forceAction))
 			}
@@ -107,7 +110,8 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, input []byte, yes, dry
 				return fmt.Errorf("postoverflow %s not found", postoverflow)
 			}
 
-			plan.AddCommand(hubops.NewDownloadCommand(item, forceAction))
+			plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, forceAction))
+
 			if !downloadOnly {
 				plan.AddCommand(hubops.NewEnableCommand(item, forceAction))
 			}