mirrors/crowdsec
1
0
Fork 0
mirror of https://github.com/crowdsecurity/crowdsec.git synced 2025-05-10 20:05:55 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

pkg/cwhub - refact Downloader (#3382)

Browse source 
* pkg/cwhub - refact Downloader
* single method interfaces
* lint
This commit is contained in:
mmetc 2024-12-27 11:49:14 +01:00 • committed by GitHub
parent fc17c0c613
commit 78f4b85311
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
31 changed files with 307 additions and 273 deletions
Download patch file Download diff file Expand all files Collapse all files

2
cmd/crowdsec-cli/clicapi/capi.go
View file

@ -261,7 +261,7 @@ func (cli *cliCapi) newStatusCmd() *cobra.Command {
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, _ []string) error {
hub, err := require.Hub(cli.cfg(), nil, nil)
hub, err := require.Hub(cli.cfg(), nil)
if err != nil {
return err
}

2
cmd/crowdsec-cli/cliconsole/console.go
View file

@ -114,7 +114,7 @@ func (cli *cliConsole) enroll(ctx context.Context, key string, name string, over
}
}
hub, err := require.Hub(cfg, nil, nil)
hub, err := require.Hub(cfg, nil)
if err != nil {
return err
}

20
cmd/crowdsec-cli/clihub/hub.go
View file

@ -93,7 +93,7 @@ func (cli *cliHub) newListCmd() *cobra.Command {
Args: cobra.NoArgs,
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger())
hub, err := require.Hub(cli.cfg(), log.StandardLogger())
if err != nil {
return err
}
@ -110,15 +110,15 @@ func (cli *cliHub) newListCmd() *cobra.Command {
func (cli *cliHub) update(ctx context.Context, withContent bool) error {
local := cli.cfg().Hub
remote := require.RemoteHub(ctx, cli.cfg())
// don't use require.Hub because if there is no index file, it would fail
hub, err := cwhub.NewHub(local, remote, log.StandardLogger())
hub, err := cwhub.NewHub(local, log.StandardLogger())
if err != nil {
return err
}
if err := hub.Update(ctx, withContent); err != nil {
indexProvider := require.HubDownloader(ctx, cli.cfg())
if err := hub.Update(ctx, indexProvider, withContent); err != nil {
return fmt.Errorf("failed to update hub: %w", err)
}
@ -166,16 +166,18 @@ cscli hub update --with-content`,
func (cli *cliHub) upgrade(ctx context.Context, yes bool, dryRun bool, force bool) error {
cfg := cli.cfg()
hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger())
hub, err := require.Hub(cfg, log.StandardLogger())
if err != nil {
return err
}
plan := hubops.NewActionPlan(hub)
contentProvider := require.HubDownloader(ctx, cfg)
for _, itemType := range cwhub.ItemTypes {
for _, item := range hub.GetInstalledByType(itemType, true) {
plan.AddCommand(hubops.NewDownloadCommand(item, force))
plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force))
}
}
@ -196,9 +198,9 @@ func (cli *cliHub) upgrade(ctx context.Context, yes bool, dryRun bool, force boo
func (cli *cliHub) newUpgradeCmd() *cobra.Command {
var (
yes bool
yes bool
dryRun bool
force bool
force bool
)
cmd := &cobra.Command{

53
cmd/crowdsec-cli/cliitem/item.go
View file

@ -71,13 +71,15 @@ func (cli cliItem) NewCommand() *cobra.Command {
func (cli cliItem) install(ctx context.Context, args []string, yes bool, dryRun bool, downloadOnly bool, force bool, ignoreError bool) error {
cfg := cli.cfg()
hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger())
hub, err := require.Hub(cfg, log.StandardLogger())
if err != nil {
return err
}
plan := hubops.NewActionPlan(hub)
contentProvider := require.HubDownloader(ctx, cfg)
for _, name := range args {
item := hub.GetItem(cli.name, name)
if item == nil {
@ -91,7 +93,7 @@ func (cli cliItem) install(ctx context.Context, args []string, yes bool, dryRun
continue
}
if err = plan.AddCommand(hubops.NewDownloadCommand(item, force)); err != nil {
if err = plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)); err != nil {
return err
}
@ -121,7 +123,7 @@ func (cli cliItem) install(ctx context.Context, args []string, yes bool, dryRun
func (cli cliItem) newInstallCmd() *cobra.Command {
var (
yes bool
yes bool
dryRun bool
downloadOnly bool
force bool
@ -180,6 +182,7 @@ func (cli cliItem) removePlan(hub *cwhub.Hub, args []string, purge bool, force b
if err := plan.AddCommand(hubops.NewDisableCommand(item, force)); err != nil {
return nil, err
}
if purge {
if err := plan.AddCommand(hubops.NewPurgeCommand(item, force)); err != nil {
return nil, err
@ -211,12 +214,11 @@ func (cli cliItem) removePlan(hub *cwhub.Hub, args []string, purge bool, force b
if err := plan.AddCommand(hubops.NewDisableCommand(item, force)); err != nil {
return nil, err
}
if purge {
if err := plan.AddCommand(hubops.NewPurgeCommand(item, force)); err != nil {
return nil, err
}
}
}
@ -224,11 +226,10 @@ func (cli cliItem) removePlan(hub *cwhub.Hub, args []string, purge bool, force b
return plan, nil
}
func (cli cliItem) remove(ctx context.Context, args []string, yes bool, dryRun bool, purge bool, force bool, all bool) error {
cfg := cli.cfg()
hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger())
hub, err := require.Hub(cli.cfg(), log.StandardLogger())
if err != nil {
return err
}
@ -253,7 +254,7 @@ func (cli cliItem) remove(ctx context.Context, args []string, yes bool, dryRun b
func (cli cliItem) newRemoveCmd() *cobra.Command {
var (
yes bool
yes bool
dryRun bool
purge bool
force bool
@ -290,12 +291,12 @@ func (cli cliItem) newRemoveCmd() *cobra.Command {
return cmd
}
func (cli cliItem) upgradePlan(hub *cwhub.Hub, args []string, force bool, all bool) (*hubops.ActionPlan, error) {
func (cli cliItem) upgradePlan(hub *cwhub.Hub, contentProvider cwhub.ContentProvider, args []string, force bool, all bool) (*hubops.ActionPlan, error) {
plan := hubops.NewActionPlan(hub)
if all {
for _, item := range hub.GetInstalledByType(cli.name, true) {
if err := plan.AddCommand(hubops.NewDownloadCommand(item, force)); err != nil {
if err := plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)); err != nil {
return nil, err
}
}
@ -313,7 +314,7 @@ func (cli cliItem) upgradePlan(hub *cwhub.Hub, args []string, force bool, all bo
return nil, fmt.Errorf("can't find '%s' in %s", itemName, cli.name)
}
if err := plan.AddCommand(hubops.NewDownloadCommand(item, force)); err != nil {
if err := plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, force)); err != nil {
return nil, err
}
}
@ -324,12 +325,14 @@ func (cli cliItem) upgradePlan(hub *cwhub.Hub, args []string, force bool, all bo
func (cli cliItem) upgrade(ctx context.Context, args []string, yes bool, dryRun bool, force bool, all bool) error {
cfg := cli.cfg()
hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger())
hub, err := require.Hub(cfg, log.StandardLogger())
if err != nil {
return err
}
plan, err := cli.upgradePlan(hub, args, force, all)
contentProvider := require.HubDownloader(ctx, cfg)
plan, err := cli.upgradePlan(hub, contentProvider, args, force, all)
if err != nil {
return err
}
@ -349,10 +352,10 @@ func (cli cliItem) upgrade(ctx context.Context, args []string, yes bool, dryRun
func (cli cliItem) newUpgradeCmd() *cobra.Command {
var (
yes bool
yes bool
dryRun bool
all bool
force bool
all bool
force bool
)
cmd := &cobra.Command{
@ -390,13 +393,13 @@ func (cli cliItem) inspect(ctx context.Context, args []string, url string, diff
cfg.Cscli.PrometheusUrl = url
}
remote := (*cwhub.RemoteHubCfg)(nil)
var contentProvider cwhub.ContentProvider
if diff {
remote = require.RemoteHub(ctx, cfg)
contentProvider = require.HubDownloader(ctx, cfg)
}
hub, err := require.Hub(cfg, remote, log.StandardLogger())
hub, err := require.Hub(cfg, log.StandardLogger())
if err != nil {
return err
}
@ -408,7 +411,7 @@ func (cli cliItem) inspect(ctx context.Context, args []string, url string, diff
}
if diff {
fmt.Println(cli.whyTainted(ctx, hub, item, rev))
fmt.Println(cli.whyTainted(ctx, hub, contentProvider, item, rev))
continue
}
@ -462,7 +465,7 @@ func (cli cliItem) newInspectCmd() *cobra.Command {
func (cli cliItem) list(args []string, all bool) error {
cfg := cli.cfg()
hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger())
hub, err := require.Hub(cli.cfg(), log.StandardLogger())
if err != nil {
return err
}
@ -498,7 +501,7 @@ func (cli cliItem) newListCmd() *cobra.Command {
}
// return the diff between the installed version and the latest version
func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, reverse bool) (string, error) {
func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, contentProvider cwhub.ContentProvider, reverse bool) (string, error) {
if !item.State.Installed {
return "", fmt.Errorf("'%s' is not installed", item.FQName())
}
@ -509,7 +512,7 @@ func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, reverse bool)
}
defer os.Remove(dest.Name())
_, remoteURL, err := item.FetchContentTo(ctx, dest.Name())
_, remoteURL, err := item.FetchContentTo(ctx, contentProvider, dest.Name())
if err != nil {
return "", err
}
@ -540,7 +543,7 @@ func (cli cliItem) itemDiff(ctx context.Context, item *cwhub.Item, reverse bool)
return fmt.Sprintf("%s", diff), nil
}
func (cli cliItem) whyTainted(ctx context.Context, hub *cwhub.Hub, item *cwhub.Item, reverse bool) string {
func (cli cliItem) whyTainted(ctx context.Context, hub *cwhub.Hub, contentProvider cwhub.ContentProvider, item *cwhub.Item, reverse bool) string {
if !item.State.Installed {
return fmt.Sprintf("# %s is not installed", item.FQName())
}
@ -565,7 +568,7 @@ func (cli cliItem) whyTainted(ctx context.Context, hub *cwhub.Hub, item *cwhub.I
ret = append(ret, err.Error())
}
diff, err := cli.itemDiff(ctx, sub, reverse)
diff, err := cli.itemDiff(ctx, sub, contentProvider, reverse)
if err != nil {
ret = append(ret, err.Error())
}

4
cmd/crowdsec-cli/cliitem/suggest.go
View file

@ -37,7 +37,7 @@ func suggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) str
}
func compAllItems(itemType string, args []string, toComplete string, cfg configGetter) ([]string, cobra.ShellCompDirective) {
hub, err := require.Hub(cfg(), nil, nil)
hub, err := require.Hub(cfg(), nil)
if err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
@ -56,7 +56,7 @@ func compAllItems(itemType string, args []string, toComplete string, cfg configG
}
func compInstalledItems(itemType string, args []string, toComplete string, cfg configGetter) ([]string, cobra.ShellCompDirective) {
hub, err := require.Hub(cfg(), nil, nil)
hub, err := require.Hub(cfg(), nil)
if err != nil {
return nil, cobra.ShellCompDirectiveDefault
}

6
cmd/crowdsec-cli/clilapi/context.go
View file

@ -59,7 +59,7 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
`,
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
hub, err := require.Hub(cli.cfg(), nil, nil)
hub, err := require.Hub(cli.cfg(), nil)
if err != nil {
return err
}
@ -101,7 +101,7 @@ func (cli *cliLapi) newContextStatusCmd() *cobra.Command {
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
hub, err := require.Hub(cfg, nil, nil)
hub, err := require.Hub(cfg, nil)
if err != nil {
return err
}
@ -153,7 +153,7 @@ cscli lapi context detect crowdsecurity/sshd-logs
return fmt.Errorf("failed to init expr helpers: %w", err)
}
hub, err := require.Hub(cfg, nil, nil)
hub, err := require.Hub(cfg, nil)
if err != nil {
return err
}

2
cmd/crowdsec-cli/clilapi/status.go
View file

@ -102,7 +102,7 @@ func (cli *cliLapi) newStatusCmd() *cobra.Command {
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, _ []string) error {
hub, err := require.Hub(cli.cfg(), nil, nil)
hub, err := require.Hub(cli.cfg(), nil)
if err != nil {
return err
}

8
cmd/crowdsec-cli/clisetup/setup.go
View file

@ -95,7 +95,7 @@ func (cli *cliSetup) newDetectCmd() *cobra.Command {
func (cli *cliSetup) newInstallHubCmd() *cobra.Command {
var (
yes bool
yes bool
dryRun bool
)
@ -289,14 +289,16 @@ func (cli *cliSetup) install(ctx context.Context, yes bool, dryRun bool, fromFil
cfg := cli.cfg()
hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), log.StandardLogger())
hub, err := require.Hub(cfg, log.StandardLogger())
if err != nil {
return err
}
verbose := (cfg.Cscli.Output == "raw")
return setup.InstallHubItems(ctx, hub, input, yes, dryRun, verbose)
contentProvider := require.HubDownloader(ctx, cfg)
return setup.InstallHubItems(ctx, hub, contentProvider, input, yes, dryRun, verbose)
}
func (cli *cliSetup) validate(fromFile string) error {

2
cmd/crowdsec-cli/clisimulation/simulation.go
View file

@ -71,7 +71,7 @@ func (cli *cliSimulation) newEnableCmd() *cobra.Command {
Example: `cscli simulation enable`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
hub, err := require.Hub(cli.cfg(), nil, nil)
hub, err := require.Hub(cli.cfg(), nil)
if err != nil {
return err
}

4
cmd/crowdsec-cli/clisupport/support.go
View file

@ -491,9 +491,9 @@ func (cli *cliSupport) dump(ctx context.Context, outFile string) error {
skipAgent = true
}
hub, err := require.Hub(cfg, nil, nil)
hub, err := require.Hub(cfg, nil)
if err != nil {
log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected")
log.Warn("Could not init hub, running on LAPI? Hub related information will not be collected")
// XXX: lapi status check requires scenarios, will return an error
}

2
cmd/crowdsec-cli/config_backup.go
View file

@ -15,7 +15,7 @@ import (
)
func (cli *cliConfig) backupHub(dirPath string) error {
hub, err := require.Hub(cli.cfg(), nil, nil)
hub, err := require.Hub(cli.cfg(), nil)
if err != nil {
return err
}

6
cmd/crowdsec-cli/config_restore.go
View file

@ -18,11 +18,13 @@ import (
func (cli *cliConfig) restoreHub(ctx context.Context, dirPath string) error {
cfg := cli.cfg()
hub, err := require.Hub(cfg, require.RemoteHub(ctx, cfg), nil)
hub, err := require.Hub(cfg, nil)
if err != nil {
return err
}
contentProvider := require.HubDownloader(ctx, cfg)
for _, itype := range cwhub.ItemTypes {
itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
if _, err = os.Stat(itemDirectory); err != nil {
@ -53,7 +55,7 @@ func (cli *cliConfig) restoreHub(ctx context.Context, dirPath string) error {
plan := hubops.NewActionPlan(hub)
if err = plan.AddCommand(hubops.NewDownloadCommand(item, false)); err != nil {
if err = plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, false)); err != nil {
return err
}

9
cmd/crowdsec-cli/require/require.go
View file

@ -82,12 +82,11 @@ func Notifications(c *csconfig.Config) error {
return nil
}
// RemoteHub returns the configuration required to download hub index and items: url, branch, etc.
func RemoteHub(ctx context.Context, c *csconfig.Config) *cwhub.RemoteHubCfg {
func HubDownloader(ctx context.Context, c *csconfig.Config) *cwhub.Downloader {
// set branch in config, and log if necessary
branch := HubBranch(ctx, c)
urlTemplate := HubURLTemplate(c)
remote := &cwhub.RemoteHubCfg{
remote := &cwhub.Downloader{
Branch: branch,
URLTemplate: urlTemplate,
IndexPath: ".index.json",
@ -98,7 +97,7 @@ func RemoteHub(ctx context.Context, c *csconfig.Config) *cwhub.RemoteHubCfg {
// Hub initializes the hub. If a remote configuration is provided, it can be used to download the index and items.
// If no remote parameter is provided, the hub can only be used for local operations.
func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg, logger *logrus.Logger) (*cwhub.Hub, error) {
func Hub(c *csconfig.Config, logger *logrus.Logger) (*cwhub.Hub, error) {
local := c.Hub
if local == nil {
@ -110,7 +109,7 @@ func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg, logger *logrus.Logger)
logger.SetOutput(io.Discard)
}
hub, err := cwhub.NewHub(local, remote, logger)
hub, err := cwhub.NewHub(local, logger)
if err != nil {
return nil, err
}

4
cmd/crowdsec/serve.go
View file

@ -85,7 +85,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
}
if !cConfig.DisableAgent {
hub, err := cwhub.NewHub(cConfig.Hub, nil, log.StandardLogger())
hub, err := cwhub.NewHub(cConfig.Hub, log.StandardLogger())
if err != nil {
return nil, err
}
@ -387,7 +387,7 @@ func Serve(cConfig *csconfig.Config, agentReady chan bool) error {
}
if !cConfig.DisableAgent {
hub, err := cwhub.NewHub(cConfig.Hub, nil, log.StandardLogger())
hub, err := cwhub.NewHub(cConfig.Hub, log.StandardLogger())
if err != nil {
return err
}

2
go.mod
View file

@ -29,7 +29,7 @@ require (
github.com/creack/pty v1.1.21 // indirect
github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26
github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241219154300-555e14e3988f
github.com/crowdsecurity/go-cs-lib v0.0.16
github.com/crowdsecurity/grokky v0.2.2
github.com/crowdsecurity/machineid v1.0.2
github.com/davecgh/go-spew v1.1.1

6
go.sum
View file

@ -107,10 +107,8 @@ github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607 h1:hyrYw3h
github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607/go.mod h1:br36fEqurGYZQGit+iDYsIzW0FF6VufMbDzyyLxEuPA=
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU=
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241203101722-e557f9809413 h1:VIedap4s3mXM4+tM2NMm7R3E/kn79ayLZaLHDqPYVCc=
github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241203101722-e557f9809413/go.mod h1:XwGcvTt4lMq4Tm1IRMSKMDf0CVrnytTU8Uoofa7AR+g=
github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241219154300-555e14e3988f h1:Pd+O4UK78uQtTqbvYX+nHvqZ7TffD51uC4q0RE/podk=
github.com/crowdsecurity/go-cs-lib v0.0.16-0.20241219154300-555e14e3988f/go.mod h1:XwGcvTt4lMq4Tm1IRMSKMDf0CVrnytTU8Uoofa7AR+g=
github.com/crowdsecurity/go-cs-lib v0.0.16 h1:2/htodjwc/sfsv4deX8F/2Fzg1bOI8w3O1/BPSvvsB0=
github.com/crowdsecurity/go-cs-lib v0.0.16/go.mod h1:XwGcvTt4lMq4Tm1IRMSKMDf0CVrnytTU8Uoofa7AR+g=
github.com/crowdsecurity/grokky v0.2.2 h1:yALsI9zqpDArYzmSSxfBq2dhYuGUTKMJq8KOEIAsuo4=
github.com/crowdsecurity/grokky v0.2.2/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM=
github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc=

16
pkg/cwhub/cwhub_test.go
View file

@ -57,18 +57,18 @@ func testHub(t *testing.T, update bool) *Hub {
os.RemoveAll(tmpDir)
})
remote := &RemoteHubCfg{
Branch: "master",
URLTemplate: mockURLTemplate,
IndexPath: ".index.json",
}
hub, err := NewHub(local, remote, log.StandardLogger())
hub, err := NewHub(local, log.StandardLogger())
require.NoError(t, err)
if update {
indexProvider := &Downloader{
Branch: "master",
URLTemplate: mockURLTemplate,
IndexPath: ".index.json",
}
ctx := context.Background()
err := hub.Update(ctx, false)
err := hub.Update(ctx, indexProvider, false)
require.NoError(t, err)
}

11
pkg/cwhub/doc.go
View file

@ -85,25 +85,20 @@
// return fmt.Errorf("collection not found")
// }
//
// To provide the remote hub configuration, use the second parameter of NewHub():
// Some commands require an object to provide the hub index, or contents:
//
// remoteHub := cwhub.RemoteHubCfg{
// indexProvider := cwhub.Downloader{
// URLTemplate: "https://cdn-hub.crowdsec.net/crowdsecurity/%s/%s",
// Branch: "master",
// IndexPath: ".index.json",
// }
//
// hub, err := cwhub.NewHub(localHub, remoteHub, logger)
// if err != nil {
// return fmt.Errorf("unable to initialize hub: %w", err)
// }
//
// The URLTemplate is a string that will be used to build the URL of the remote hub. It must contain two
// placeholders: the branch and the file path (it will be an index or an item).
//
// Before calling hub.Load(), you can update the index file by calling the Update() method:
//
// err := hub.Update(context.Background())
// err := hub.Update(context.Background(), indexProvider)
// if err != nil {
// return fmt.Errorf("unable to update hub index: %w", err)
// }

124
pkg/cwhub/download.go Normal file
View file

@ -0,0 +1,124 @@
package cwhub
import (
"context"
"errors"
"fmt"
"net/http"
"net/url"
"github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/downloader"
)
// Downloader is used to retrieve index and items from a remote hub, with cache control.
type Downloader struct {
Branch string
URLTemplate string
IndexPath string
}
// IndexProvider retrieves and writes .index.json
type IndexProvider interface {
FetchIndex(ctx context.Context, indexFile string, withContent bool, logger *logrus.Logger) (bool, error)
}
// ContentProvider retrieves and writes the YAML files with the item content.
type ContentProvider interface {
FetchContent(ctx context.Context, remotePath, destPath, wantHash string, logger *logrus.Logger) (bool, string, error)
}
// urlTo builds the URL to download a file from the remote hub.
func (d *Downloader) urlTo(remotePath string) (string, error) {
// the template must contain two string placeholders
if fmt.Sprintf(d.URLTemplate, "%s", "%s") != d.URLTemplate {
return "", fmt.Errorf("invalid URL template '%s'", d.URLTemplate)
}
return fmt.Sprintf(d.URLTemplate, d.Branch, remotePath), nil
}
// addURLParam adds a parameter with a value (ex. "with_content=true") to the URL if it's not already present.
func addURLParam(rawURL string, param string, value string) (string, error) {
parsedURL, err := url.Parse(rawURL)
if err != nil {
return "", fmt.Errorf("failed to parse URL: %w", err)
}
query := parsedURL.Query()
if _, exists := query[param]; !exists {
query.Add(param, value)
}
parsedURL.RawQuery = query.Encode()
return parsedURL.String(), nil
}
// FetchIndex downloads the index from the hub and writes it to the filesystem.
// It uses a temporary file to avoid partial downloads, and won't overwrite the original
// if it has not changed.
func (d *Downloader) FetchIndex(ctx context.Context, destPath string, withContent bool, logger *logrus.Logger) (bool, error) {
url, err := d.urlTo(d.IndexPath)
if err != nil {
return false, fmt.Errorf("failed to build hub index request: %w", err)
}
if withContent {
url, err = addURLParam(url, "with_content", "true")
if err != nil {
return false, fmt.Errorf("failed to add 'with_content' parameter to URL: %w", err)
}
}
downloaded, err := downloader.
New().
WithHTTPClient(HubClient).
ToFile(destPath).
WithETagFn(downloader.SHA256).
CompareContent().
WithLogger(logger.WithField("url", url)).
BeforeRequest(func(_ *http.Request) {
fmt.Println("Downloading " + destPath)
}).
Download(ctx, url)
if err != nil {
return false, err
}
return downloaded, nil
}
// FetchContent downloads the content to the specified path, through a temporary file
// to avoid partial downloads.
// If the hash does not match, it will not overwrite and log a warning.
func (d *Downloader) FetchContent(ctx context.Context, remotePath, destPath, wantHash string, logger *logrus.Logger) (bool, string, error) {
url, err := d.urlTo(remotePath)
if err != nil {
return false, "", fmt.Errorf("failed to build request: %w", err)
}
downloaded, err := downloader.
New().
WithHTTPClient(HubClient).
ToFile(destPath).
WithETagFn(downloader.SHA256).
WithMakeDirs(true).
WithLogger(logger.WithField("url", url)).
CompareContent().
VerifyHash("sha256", wantHash).
Download(ctx, url)
var hasherr downloader.HashMismatchError
switch {
case errors.As(err, &hasherr):
logger.Warnf("%s. The index file is outdated, please run 'cscli hub update' and try again", err.Error())
case err != nil:
return false, "", err
}
return downloaded, url, nil
}

50
pkg/cwhub/download_test.go Normal file
View file

@ -0,0 +1,50 @@
package cwhub
import (
"context"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestFetchIndex(t *testing.T) {
ctx := context.Background()
mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Query().Get("with_content") == "true" {
w.WriteHeader(http.StatusOK)
w.Write([]byte(`Hi I'm an index with content`))
} else {
w.WriteHeader(http.StatusOK)
w.Write([]byte(`Hi I'm a regular index`))
}
}))
defer mockServer.Close()
downloader := &Downloader{
Branch: "main",
URLTemplate: mockServer.URL + "/%s/%s",
IndexPath: "index.txt",
}
logger := logrus.New()
logger.Out = io.Discard
destPath := filepath.Join(t.TempDir(), "index.txt")
withContent := true
downloaded, err := downloader.FetchIndex(ctx, destPath, withContent, logger)
require.NoError(t, err)
assert.True(t, downloaded)
content, err := os.ReadFile(destPath)
require.NoError(t, err)
assert.Equal(t, "Hi I'm an index with content", string(content))
}

19
pkg/cwhub/errors.go
View file

@ -1,19 +0,0 @@
package cwhub
import (
"errors"
"fmt"
)
// ErrNilRemoteHub is returned when trying to download with a local-only configuration.
var ErrNilRemoteHub = errors.New("remote hub configuration is not provided. Please report this issue to the developers")
// IndexNotFoundError is returned when the remote hub index is not found.
type IndexNotFoundError struct {
URL string
Branch string
}
func (e IndexNotFoundError) Error() string {
return fmt.Sprintf("index not found at %s, branch '%s'. Please check the .cscli.hub_branch value if you specified it in config.yaml, or use 'master' if not sure", e.URL, e.Branch)
}

44
pkg/cwhub/fetch.go
View file

@ -1,24 +1,15 @@
package cwhub
// Install, upgrade and remove items from the hub to the local configuration
import (
"context"
"crypto"
"encoding/base64"
"encoding/hex"
"errors"
"fmt"
"os"
"path/filepath"
"github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/downloader"
)
// writeEmbeddedContentTo writes the embedded content to the specified path and checks the hash.
// If the content is base64 encoded, it will be decoded before writing. Check for item.Content
// before calling this method.
@ -56,40 +47,9 @@ func (i *Item) writeEmbeddedContentTo(destPath, wantHash string) error {
return nil
}
// writeRemoteContentTo downloads the content to the specified path and checks the hash.
func (i *Item) writeRemoteContentTo(ctx context.Context, destPath, wantHash string) (bool, string, error) {
url, err := i.hub.remote.urlTo(i.RemotePath)
if err != nil {
return false, "", fmt.Errorf("failed to build request: %w", err)
}
d := downloader.
New().
WithHTTPClient(HubClient).
ToFile(destPath).
WithETagFn(downloader.SHA256).
WithMakeDirs(true).
WithLogger(logrus.WithField("url", url)).
CompareContent().
VerifyHash("sha256", wantHash)
hasherr := downloader.HashMismatchError{}
downloaded, err := d.Download(ctx, url)
switch {
case errors.As(err, &hasherr):
i.hub.logger.Warnf("%s. The index file is outdated, please run 'cscli hub update' and try again", err.Error())
case err != nil:
return false, "", err
}
return downloaded, url, nil
}
// FetchContentTo writes the last version of the item's YAML file to the specified path.
// Returns whether the file was downloaded, and the remote url for feedback purposes.
func (i *Item) FetchContentTo(ctx context.Context, destPath string) (bool, string, error) {
func (i *Item) FetchContentTo(ctx context.Context, contentProvider ContentProvider, destPath string) (bool, string, error) {
wantHash := i.latestHash()
if wantHash == "" {
return false, "", fmt.Errorf("%s: latest hash missing from index. The index file is invalid, please run 'cscli hub update' and try again", i.FQName())
@ -104,5 +64,5 @@ func (i *Item) FetchContentTo(ctx context.Context, destPath string) (bool, strin
return true, fmt.Sprintf("(embedded in %s)", i.hub.local.HubIndexFile), nil
}
return i.writeRemoteContentTo(ctx, destPath, wantHash)
return contentProvider.FetchContent(ctx, i.RemotePath, destPath, wantHash, i.hub.logger)
}

11
pkg/cwhub/hub.go
View file

@ -22,7 +22,6 @@ type Hub struct {
items HubItems // Items read from HubDir and InstallDir
pathIndex map[string]*Item
local *csconfig.LocalHubCfg
remote *RemoteHubCfg
logger *logrus.Logger
Warnings []string // Warnings encountered during sync
}
@ -35,8 +34,7 @@ func (h *Hub) GetDataDir() string {
// NewHub returns a new Hub instance with local and (optionally) remote configuration.
// The hub is not synced automatically. Load() must be called to read the index, sync the local state,
// and check for unmanaged items.
// All download operations (including updateIndex) return ErrNilRemoteHub if the remote configuration is not set.
func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, logger *logrus.Logger) (*Hub, error) {
func NewHub(local *csconfig.LocalHubCfg, logger *logrus.Logger) (*Hub, error) {
if local == nil {
return nil, errors.New("no hub configuration found")
}
@ -48,7 +46,6 @@ func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, logger *logrus.Lo
hub := &Hub{
local: local,
remote: remote,
logger: logger,
pathIndex: make(map[string]*Item, 0),
}
@ -158,13 +155,13 @@ func (h *Hub) ItemStats() []string {
// Update downloads the latest version of the index and writes it to disk if it changed.
// It cannot be called after Load() unless the hub is completely empty.
func (h *Hub) Update(ctx context.Context, withContent bool) error {
func (h *Hub) Update(ctx context.Context, indexProvider IndexProvider, withContent bool) error {
if len(h.pathIndex) > 0 {
// if this happens, it's a bug.
return errors.New("cannot update hub after items have been loaded")
}
downloaded, err := h.remote.fetchIndex(ctx, h.local.HubIndexFile, withContent)
downloaded, err := indexProvider.FetchIndex(ctx, h.local.HubIndexFile, withContent, h.logger)
if err != nil {
return err
}
@ -238,6 +235,7 @@ func (h *Hub) GetItemsByType(itemType string, sorted bool) []*Item {
}
idx := 0
for _, item := range items {
ret[idx] = item
idx += 1
@ -269,6 +267,7 @@ func (h *Hub) GetInstalledListForAPI() []string {
ret := make([]string, len(scenarios)+len(appsecRules))
idx := 0
for _, item := range scenarios {
ret[idx] = item.Name
idx += 1

35
pkg/cwhub/hub_test.go
View file

@ -13,18 +13,19 @@ import (
func TestInitHubUpdate(t *testing.T) {
hub := envSetup(t)
remote := &RemoteHubCfg{
_, err := NewHub(hub.local, nil)
require.NoError(t, err)
ctx := context.Background()
indexProvider := &Downloader{
URLTemplate: mockURLTemplate,
Branch: "master",
IndexPath: ".index.json",
}
_, err := NewHub(hub.local, remote, nil)
require.NoError(t, err)
ctx := context.Background()
err = hub.Update(ctx, false)
err = hub.Update(ctx, indexProvider, false)
require.NoError(t, err)
err = hub.Load()
@ -48,29 +49,29 @@ func TestUpdateIndex(t *testing.T) {
hub := envSetup(t)
hub.remote = &RemoteHubCfg{
hub.local.HubIndexFile = tmpIndex.Name()
ctx := context.Background()
indexProvider := &Downloader{
URLTemplate: "x",
Branch: "",
IndexPath: "",
}
hub.local.HubIndexFile = tmpIndex.Name()
ctx := context.Background()
err = hub.Update(ctx, false)
err = hub.Update(ctx, indexProvider, false)
cstest.RequireErrorContains(t, err, "failed to build hub index request: invalid URL template 'x'")
// bad domain
fmt.Println("Test 'bad domain'")
hub.remote = &RemoteHubCfg{
indexProvider = &Downloader{
URLTemplate: "https://baddomain/crowdsecurity/%s/%s",
Branch: "master",
IndexPath: ".index.json",
}
err = hub.Update(ctx, false)
err = hub.Update(ctx, indexProvider, false)
require.NoError(t, err)
// XXX: this is not failing
// cstest.RequireErrorContains(t, err, "failed http request for hub index: Get")
@ -78,7 +79,7 @@ func TestUpdateIndex(t *testing.T) {
// bad target path
fmt.Println("Test 'bad target path'")
hub.remote = &RemoteHubCfg{
indexProvider = &Downloader{
URLTemplate: mockURLTemplate,
Branch: "master",
IndexPath: ".index.json",
@ -86,6 +87,6 @@ func TestUpdateIndex(t *testing.T) {
hub.local.HubIndexFile = "/does/not/exist/index.json"
err = hub.Update(ctx, false)
err = hub.Update(ctx, indexProvider, false)
cstest.RequireErrorContains(t, err, "failed to create temporary download file for /does/not/exist/index.json:")
}

8
pkg/cwhub/itemupgrade_test.go
View file

@ -38,7 +38,7 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
// collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario"
pushUpdateToCollectionInHub()
remote := &RemoteHubCfg{
remote := &Downloader{
URLTemplate: mockURLTemplate,
Branch: "master",
IndexPath: ".index.json",
@ -98,7 +98,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
require.NoError(t, err)
require.True(t, didRemove)
remote := &RemoteHubCfg{
remote := &Downloader{
URLTemplate: mockURLTemplate,
Branch: "master",
IndexPath: ".index.json",
@ -132,7 +132,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
}
// getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test.
func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCfg) *Hub {
func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *Downloader) *Hub {
hub, err := NewHub(local, remote, nil)
require.NoError(t, err)
@ -170,7 +170,7 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te
require.NoError(t, err)
require.True(t, didRemove)
remote := &RemoteHubCfg{
remote := &Downloader{
URLTemplate: mockURLTemplate,
Branch: "master",
IndexPath: ".index.json",

87
pkg/cwhub/remote.go
View file

@ -1,87 +0,0 @@
package cwhub
import (
"context"
"fmt"
"net/http"
"net/url"
"github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/downloader"
)
// RemoteHubCfg is used to retrieve index and items from the remote hub.
type RemoteHubCfg struct {
Branch string
URLTemplate string
IndexPath string
}
// urlTo builds the URL to download a file from the remote hub.
func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) {
if r == nil {
return "", ErrNilRemoteHub
}
// the template must contain two string placeholders
if fmt.Sprintf(r.URLTemplate, "%s", "%s") != r.URLTemplate {
return "", fmt.Errorf("invalid URL template '%s'", r.URLTemplate)
}
return fmt.Sprintf(r.URLTemplate, r.Branch, remotePath), nil
}
// addURLParam adds a parameter with a value (ex. "with_content=true") to the URL if it's not already present.
func addURLParam(rawURL string, param string, value string) (string, error) {
parsedURL, err := url.Parse(rawURL)
if err != nil {
return "", fmt.Errorf("failed to parse URL: %w", err)
}
query := parsedURL.Query()
if _, exists := query[param]; !exists {
query.Add(param, value)
}
parsedURL.RawQuery = query.Encode()
return parsedURL.String(), nil
}
// fetchIndex downloads the index from the hub and returns the content.
func (r *RemoteHubCfg) fetchIndex(ctx context.Context, destPath string, withContent bool) (bool, error) {
if r == nil {
return false, ErrNilRemoteHub
}
url, err := r.urlTo(r.IndexPath)
if err != nil {
return false, fmt.Errorf("failed to build hub index request: %w", err)
}
if withContent {
url, err = addURLParam(url, "with_content", "true")
if err != nil {
return false, fmt.Errorf("failed to add 'with_content' parameter to URL: %w", err)
}
}
downloaded, err := downloader.
New().
WithHTTPClient(HubClient).
ToFile(destPath).
WithETagFn(downloader.SHA256).
CompareContent().
WithLogger(logrus.WithField("url", url)).
BeforeRequest(func(_ *http.Request) {
fmt.Println("Downloading "+destPath)
}).
Download(ctx, url)
if err != nil {
return false, err
}
return downloaded, nil
}

16
pkg/hubops/download.go
View file

@ -9,9 +9,9 @@ import (
"os"
"time"
"github.com/fatih/color"
"github.com/sirupsen/logrus"
"gopkg.in/yaml.v3"
"github.com/fatih/color"
"github.com/crowdsecurity/go-cs-lib/downloader"
@ -19,19 +19,19 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/types"
)
// DownloadCommand handles the downloading of hub items.
// It ensures that items are fetched from the hub (or from the index file if it also has content)
// managing dependencies and verifying the integrity of downloaded content.
// This is used by "cscli install" and "cscli upgrade".
// Tainted items require the force parameter, local items are skipped.
type DownloadCommand struct {
Item *cwhub.Item
Force bool
Item *cwhub.Item
Force bool
contentProvider cwhub.ContentProvider
}
func NewDownloadCommand(item *cwhub.Item, force bool) *DownloadCommand {
return &DownloadCommand{Item: item, Force: force}
func NewDownloadCommand(item *cwhub.Item, contentProvider cwhub.ContentProvider, force bool) *DownloadCommand {
return &DownloadCommand{Item: item, Force: force, contentProvider: contentProvider}
}
func (c *DownloadCommand) Prepare(plan *ActionPlan) (bool, error) {
@ -60,7 +60,7 @@ func (c *DownloadCommand) Prepare(plan *ActionPlan) (bool, error) {
}
for sub := range i.LatestDependencies().SubItems(plan.hub) {
if err := plan.AddCommand(NewDownloadCommand(sub, c.Force)); err != nil {
if err := plan.AddCommand(NewDownloadCommand(sub, c.contentProvider, c.Force)); err != nil {
return false, err
}
@ -158,7 +158,7 @@ func (c *DownloadCommand) Run(ctx context.Context, plan *ActionPlan) error {
return err
}
downloaded, _, err := i.FetchContentTo(ctx, finalPath)
downloaded, _, err := i.FetchContentTo(ctx, c.contentProvider, finalPath)
if err != nil {
return fmt.Errorf("%s: %w", i.FQName(), err)
}

8
pkg/hubtest/hubtest.go
View file

@ -14,8 +14,8 @@ type HubTest struct {
CrowdSecPath string
CscliPath string
HubPath string
HubTestPath string //generic parser/scenario tests .tests
HubAppsecTestPath string //dir specific to appsec tests .appsec-tests
HubTestPath string // generic parser/scenario tests .tests
HubAppsecTestPath string // dir specific to appsec tests .appsec-tests
HubIndexFile string
TemplateConfigPath string
TemplateProfilePath string
@ -93,7 +93,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecT
InstallDataDir: HubTestPath,
}
hub, err := cwhub.NewHub(local, nil, nil)
hub, err := cwhub.NewHub(local, nil)
if err != nil {
return HubTest{}, err
}
@ -130,7 +130,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecT
InstallDataDir: HubTestPath,
}
hub, err := cwhub.NewHub(local, nil, nil)
hub, err := cwhub.NewHub(local, nil)
if err != nil {
return HubTest{}, err
}

2
pkg/hubtest/hubtest_item.go
View file

@ -212,7 +212,7 @@ func (t *HubTestItem) InstallHub() error {
}
// load installed hub
hub, err := cwhub.NewHub(t.RuntimeHubConfig, nil, nil)
hub, err := cwhub.NewHub(t.RuntimeHubConfig, nil)
if err != nil {
return err
}

3
pkg/leakybucket/buckets_test.go
View file

@ -46,7 +46,7 @@ func TestBucket(t *testing.T) {
InstallDataDir: testdata,
}
hub, err := cwhub.NewHub(hubCfg, nil, nil)
hub, err := cwhub.NewHub(hubCfg, nil)
require.NoError(t, err)
err = hub.Load()
@ -140,6 +140,7 @@ func testOneBucket(t *testing.T, hub *cwhub.Hub, dir string, tomb *tomb.Tomb) er
}
scenarios := []*cwhub.Item{}
for _, x := range stages {
// XXX: LoadBuckets should take an interface, BucketProvider ScenarioProvider or w/e
item := &cwhub.Item{

14
pkg/setup/install.go
View file

@ -48,7 +48,7 @@ func decodeSetup(input []byte, fancyErrors bool) (Setup, error) {
}
// InstallHubItems installs the objects recommended in a setup file.
func InstallHubItems(ctx context.Context, hub *cwhub.Hub, input []byte, yes, dryRun, verbose bool) error {
func InstallHubItems(ctx context.Context, hub *cwhub.Hub, contentProvider cwhub.ContentProvider, input []byte, yes, dryRun, verbose bool) error {
setupEnvelope, err := decodeSetup(input, false)
if err != nil {
return err
@ -71,7 +71,8 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, input []byte, yes, dry
return fmt.Errorf("collection %s not found", collection)
}
plan.AddCommand(hubops.NewDownloadCommand(item, forceAction))
plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, forceAction))
if !downloadOnly {
plan.AddCommand(hubops.NewEnableCommand(item, forceAction))
}
@ -83,7 +84,8 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, input []byte, yes, dry
return fmt.Errorf("parser %s not found", parser)
}
plan.AddCommand(hubops.NewDownloadCommand(item, forceAction))
plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, forceAction))
if !downloadOnly {
plan.AddCommand(hubops.NewEnableCommand(item, forceAction))
}
@ -95,7 +97,8 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, input []byte, yes, dry
return fmt.Errorf("scenario %s not found", scenario)
}
plan.AddCommand(hubops.NewDownloadCommand(item, forceAction))
plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, forceAction))
if !downloadOnly {
plan.AddCommand(hubops.NewEnableCommand(item, forceAction))
}
@ -107,7 +110,8 @@ func InstallHubItems(ctx context.Context, hub *cwhub.Hub, input []byte, yes, dry
return fmt.Errorf("postoverflow %s not found", postoverflow)
}
plan.AddCommand(hubops.NewDownloadCommand(item, forceAction))
plan.AddCommand(hubops.NewDownloadCommand(item, contentProvider, forceAction))
if !downloadOnly {
plan.AddCommand(hubops.NewEnableCommand(item, forceAction))
}