diff --git a/cmd/crowdsec-cli/collections.go b/cmd/crowdsec-cli/collections.go deleted file mode 100644 index 494cedab2..000000000 --- a/cmd/crowdsec-cli/collections.go +++ /dev/null @@ -1,333 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/fatih/color" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" -) - -func NewCollectionsCmd() *cobra.Command { - cmdCollections := &cobra.Command{ - Use: "collections [collection]...", - Short: "Manage hub collections", - Example: `cscli collections list -a -cscli collections install crowdsecurity/http-cve crowdsecurity/iptables -cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables -cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables -cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables -`, - Args: cobra.MinimumNArgs(1), - Aliases: []string{"collection"}, - DisableAutoGenTag: true, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if _, err := require.Hub(csConfig); err != nil { - return err - } - - return nil - }, - PersistentPostRun: func(cmd *cobra.Command, args []string) { - if cmd.Name() == "inspect" || cmd.Name() == "list" { - return - } - log.Infof(ReloadMessage()) - }, - } - - cmdCollections.AddCommand(NewCollectionsInstallCmd()) - cmdCollections.AddCommand(NewCollectionsRemoveCmd()) - cmdCollections.AddCommand(NewCollectionsUpgradeCmd()) - cmdCollections.AddCommand(NewCollectionsInspectCmd()) - cmdCollections.AddCommand(NewCollectionsListCmd()) - - return cmdCollections -} - -func runCollectionsInstall(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - downloadOnly, err := flags.GetBool("download-only") - if err != nil { - return err - } - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - ignoreError, err := flags.GetBool("ignore") - if err != nil { - return err - } - - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - for _, name := range args { - t := hub.GetItem(cwhub.COLLECTIONS, name) - if t == nil { - nearestItem, score := GetDistance(cwhub.COLLECTIONS, name) - Suggest(cwhub.COLLECTIONS, name, nearestItem.Name, score, ignoreError) - - continue - } - - if err := hub.InstallItem(name, cwhub.COLLECTIONS, force, downloadOnly); err != nil { - if !ignoreError { - return fmt.Errorf("error while installing '%s': %w", name, err) - } - log.Errorf("Error while installing '%s': %s", name, err) - } - } - - return nil -} - -func NewCollectionsInstallCmd() *cobra.Command { - cmdCollectionsInstall := &cobra.Command{ - Use: "install ...", - Short: "Install given collection(s)", - Long: `Fetch and install one or more collections from hub`, - Example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compAllItems(cwhub.COLLECTIONS, args, toComplete) - }, - RunE: runCollectionsInstall, - } - - flags := cmdCollectionsInstall.Flags() - flags.BoolP("download-only", "d", false, "Only download packages, don't enable") - flags.Bool("force", false, "Force install: overwrite tainted and outdated files") - flags.Bool("ignore", false, "Ignore errors when installing multiple collections") - - return cmdCollectionsInstall -} - -func runCollectionsRemove(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - purge, err := flags.GetBool("purge") - if err != nil { - return err - } - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - if all { - err := hub.RemoveMany(cwhub.COLLECTIONS, "", all, purge, force) - if err != nil { - return err - } - - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one collection to remove or '--all'") - } - - for _, name := range args { - if !force { - item := hub.GetItem(cwhub.COLLECTIONS, name) - if item == nil { - // XXX: this should be in GetItem? - return fmt.Errorf("can't find '%s' in %s", name, cwhub.COLLECTIONS) - } - if len(item.BelongsToCollections) > 0 { - log.Warningf("%s belongs to other collections: %s", name, item.BelongsToCollections) - log.Warningf("Run 'sudo cscli collections remove %s --force' if you want to force remove this sub collection", name) - continue - } - } - - err := hub.RemoveMany(cwhub.COLLECTIONS, name, all, purge, force) - if err != nil { - return err - } - } - - return nil -} - -func NewCollectionsRemoveCmd() *cobra.Command { - cmdCollectionsRemove := &cobra.Command{ - Use: "remove ...", - Short: "Remove given collection(s)", - Long: `Remove one or more collections`, - Example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`, - Aliases: []string{"delete"}, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.COLLECTIONS, args, toComplete) - }, - RunE: runCollectionsRemove, - } - - flags := cmdCollectionsRemove.Flags() - flags.Bool("purge", false, "Delete source file too") - flags.Bool("force", false, "Force remove: remove tainted and outdated files") - flags.Bool("all", false, "Remove all the collections") - - return cmdCollectionsRemove -} - -func runCollectionsUpgrade(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - if all { - if err := hub.UpgradeConfig(cwhub.COLLECTIONS, "", force); err != nil { - return err - } - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one collection to upgrade or '--all'") - } - - for _, name := range args { - if err := hub.UpgradeConfig(cwhub.COLLECTIONS, name, force); err != nil { - return err - } - } - - return nil -} - -func NewCollectionsUpgradeCmd() *cobra.Command { - cmdCollectionsUpgrade := &cobra.Command{ - Use: "upgrade ...", - Short: "Upgrade given collection(s)", - Long: `Fetch and upgrade one or more collections from the hub`, - Example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.COLLECTIONS, args, toComplete) - }, - RunE: runCollectionsUpgrade, - } - - flags := cmdCollectionsUpgrade.Flags() - flags.BoolP("all", "a", false, "Upgrade all the collections") - flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") - - return cmdCollectionsUpgrade -} - -func runCollectionsInspect(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - url, err := flags.GetString("url") - if err != nil { - return err - } - - if url != "" { - csConfig.Cscli.PrometheusUrl = url - } - - noMetrics, err := flags.GetBool("no-metrics") - if err != nil { - return err - } - - for _, name := range args { - if err = InspectItem(name, cwhub.COLLECTIONS, noMetrics); err != nil { - return err - } - } - - return nil -} - -func NewCollectionsInspectCmd() *cobra.Command { - cmdCollectionsInspect := &cobra.Command{ - Use: "inspect ...", - Short: "Inspect given collection(s)", - Long: `Inspect one or more collections`, - Example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.COLLECTIONS, args, toComplete) - }, - RunE: runCollectionsInspect, - } - - flags := cmdCollectionsInspect.Flags() - flags.StringP("url", "u", "", "Prometheus url") - flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") - - return cmdCollectionsInspect -} - -func runCollectionsList(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - if err = ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all); err != nil { - return err - } - - return nil -} - -func NewCollectionsListCmd() *cobra.Command { - cmdCollectionsList := &cobra.Command{ - Use: "list [collection... | -a]", - Short: "List collections", - Long: `List of installed/available/specified collections`, - Example: `cscli collections list -cscli collections list -a -cscli collections list crowdsecurity/http-cve crowdsecurity/iptables`, - DisableAutoGenTag: true, - RunE: runCollectionsList, - } - - flags := cmdCollectionsList.Flags() - flags.BoolP("all", "a", false, "List disabled items as well") - - return cmdCollectionsList -} diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index ba734461e..1a081d6c5 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -36,7 +36,7 @@ func silentInstallItem(name string, obtype string) (string, error) { if err != nil { return "", fmt.Errorf("error while downloading %s : %v", item.Name, err) } - if err := hub.AddItem(obtype, *item); err != nil { + if err := hub.AddItem(*item); err != nil { return "", err } @@ -44,7 +44,7 @@ func silentInstallItem(name string, obtype string) (string, error) { if err != nil { return "", fmt.Errorf("error while enabling %s : %v", item.Name, err) } - if err := hub.AddItem(obtype, *item); err != nil { + if err := hub.AddItem(*item); err != nil { return "", err } return fmt.Sprintf("Enabled %s", item.Name), nil diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index c4ec140e9..c0f69ea87 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -61,7 +61,9 @@ func runHubList(cmd *cobra.Command, args []string) error { log.Info(v) } - cwhub.DisplaySummary() + for line := range hub.ItemStats() { + log.Info(line) + } err = ListItems(color.Output, []string{ cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.POSTOVERFLOWS, diff --git a/cmd/crowdsec-cli/itemcommands.go b/cmd/crowdsec-cli/itemcommands.go new file mode 100644 index 000000000..0afa27787 --- /dev/null +++ b/cmd/crowdsec-cli/itemcommands.go @@ -0,0 +1,503 @@ +package main + +import ( + "fmt" + + "github.com/fatih/color" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/crowdsecurity/go-cs-lib/coalesce" + + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +type cmdHelp struct { + // Example is required, the others have a default value + // generated from the item type + use string + short string + long string + example string +} + +type hubItemType struct { + name string // plural, as used in the hub index + singular string + oneOrMore string // parenthetical pluralizaion: "parser(s)" + help cmdHelp + installHelp cmdHelp + removeHelp cmdHelp + upgradeHelp cmdHelp + inspectHelp cmdHelp + listHelp cmdHelp +} + +var hubItemTypes = map[string]hubItemType{ + "parsers": { + name: "parsers", + singular: "parser", + oneOrMore: "parser(s)", + help: cmdHelp{ + example: `cscli parsers list -a +cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs +`, + }, + installHelp: cmdHelp{ + example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, + }, + removeHelp: cmdHelp{ + example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, + }, + upgradeHelp: cmdHelp{ + example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, + }, + inspectHelp: cmdHelp{ + example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`, + }, + listHelp: cmdHelp{ + example: `cscli parsers list +cscli parsers list -a +cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, + }, + }, + "postoverflows": { + name: "postoverflows", + singular: "postoverflow", + oneOrMore: "postoverflow(s)", + help: cmdHelp{ + example: `cscli postoverflows list -a +cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns +`, + }, + installHelp: cmdHelp{ + example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`, + }, + removeHelp: cmdHelp{ + example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`, + }, + upgradeHelp: cmdHelp{ + example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`, + }, + inspectHelp: cmdHelp{ + example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`, + }, + listHelp: cmdHelp{ + example: `cscli postoverflows list +cscli postoverflows list -a +cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns`, + }, + }, + "scenarios": { + name: "scenarios", + singular: "scenario", + oneOrMore: "scenario(s)", + help: cmdHelp{ + example: `cscli scenarios list -a +cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing +cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing +cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing +cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing +`, + }, + installHelp: cmdHelp{ + example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`, + }, + removeHelp: cmdHelp{ + example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`, + }, + upgradeHelp: cmdHelp{ + example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`, + }, + inspectHelp: cmdHelp{ + example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`, + }, + listHelp: cmdHelp{ + example: `cscli scenarios list +cscli scenarios list -a +cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing`, + }, + }, + "collections": { + name: "collections", + singular: "collection", + oneOrMore: "collection(s)", + help: cmdHelp{ + example: `cscli collections list -a +cscli collections install crowdsecurity/http-cve crowdsecurity/iptables +cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables +cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables +cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables +`, + }, + installHelp: cmdHelp{ + example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`, + }, + removeHelp: cmdHelp{ + example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`, + }, + upgradeHelp: cmdHelp{ + example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`, + }, + inspectHelp: cmdHelp{ + example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`, + }, + listHelp: cmdHelp{ + example: `cscli collections list +cscli collections list -a +cscli collections list crowdsecurity/http-cve crowdsecurity/iptables`, + }, + }, +} + +func NewItemsCmd(typeName string) *cobra.Command { + it := hubItemTypes[typeName] + + cmd := &cobra.Command{ + Use: coalesce.String(it.help.use, fmt.Sprintf("%s [item]...", it.name)), + Short: coalesce.String(it.help.short, fmt.Sprintf("Manage hub %s", it.name)), + Long: it.help.long, + Example: it.help.example, + Args: cobra.MinimumNArgs(1), + Aliases: []string{it.singular}, + DisableAutoGenTag: true, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + if _, err := require.Hub(csConfig); err != nil { + return err + } + + return nil + }, + PersistentPostRun: func(cmd *cobra.Command, args []string) { + if cmd.Name() == "inspect" || cmd.Name() == "list" { + return + } + log.Infof(ReloadMessage()) + }, + } + + cmd.AddCommand(NewItemsInstallCmd(typeName)) + cmd.AddCommand(NewItemsRemoveCmd(typeName)) + cmd.AddCommand(NewItemsUpgradeCmd(typeName)) + cmd.AddCommand(NewItemsInspectCmd(typeName)) + cmd.AddCommand(NewItemsListCmd(typeName)) + + return cmd +} + +func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { + run := func(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + downloadOnly, err := flags.GetBool("download-only") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + ignoreError, err := flags.GetBool("ignore") + if err != nil { + return err + } + + hub, err := cwhub.GetHub() + if err != nil { + return err + } + + for _, name := range args { + t := hub.GetItem(it.name, name) + if t == nil { + nearestItem, score := GetDistance(it.name, name) + Suggest(it.name, name, nearestItem.Name, score, ignoreError) + + continue + } + + if err := hub.InstallItem(name, it.name, force, downloadOnly); err != nil { + if !ignoreError { + return fmt.Errorf("error while installing '%s': %w", name, err) + } + log.Errorf("Error while installing '%s': %s", name, err) + } + } + return nil + } + + return run +} + +func NewItemsInstallCmd(typeName string) *cobra.Command { + it := hubItemTypes[typeName] + + cmd := &cobra.Command{ + Use: coalesce.String(it.installHelp.use, "install [item]..."), + Short: coalesce.String(it.installHelp.short, fmt.Sprintf("Install given %s", it.oneOrMore)), + Long: coalesce.String(it.installHelp.long, fmt.Sprintf("Fetch and install one or more %s from the hub", it.name)), + Example: it.installHelp.example, + Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compAllItems(typeName, args, toComplete) + }, + RunE: itemsInstallRunner(it), + } + + flags := cmd.Flags() + flags.BoolP("download-only", "d", false, "Only download packages, don't enable") + flags.Bool("force", false, "Force install: overwrite tainted and outdated files") + flags.Bool("ignore", false, fmt.Sprintf("Ignore errors when installing multiple %s", it.name)) + + return cmd +} + +func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { + run := func(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + purge, err := flags.GetBool("purge") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + hub, err := cwhub.GetHub() + if err != nil { + return err + } + + if all { + err := hub.RemoveMany(it.name, "", all, purge, force) + if err != nil { + return err + } + + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one %s to remove or '--all'", it.singular) + } + + for _, name := range args { + if !force { + item := hub.GetItem(it.name, name) + if item == nil { + // XXX: this should be in GetItem? + return fmt.Errorf("can't find '%s' in %s", name, it.name) + } + if len(item.BelongsToCollections) > 0 { + log.Warningf("%s belongs to collections: %s", name, item.BelongsToCollections) + log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", it.name, name, it.singular) + continue + } + } + + err := hub.RemoveMany(it.name, name, all, purge, force) + if err != nil { + return err + } + } + + return nil + } + return run +} + +func NewItemsRemoveCmd(typeName string) *cobra.Command { + it := hubItemTypes[typeName] + + cmd := &cobra.Command{ + Use: coalesce.String(it.removeHelp.use, "remove [item]..."), + Short: coalesce.String(it.removeHelp.short, fmt.Sprintf("Remove given %s", it.oneOrMore)), + Long: coalesce.String(it.removeHelp.long, fmt.Sprintf("Remove one or more %s", it.name)), + Example: it.removeHelp.example, + Aliases: []string{"delete"}, + DisableAutoGenTag: true, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compInstalledItems(it.name, args, toComplete) + }, + RunE: itemsRemoveRunner(it), + } + + flags := cmd.Flags() + flags.Bool("purge", false, "Delete source file too") + flags.Bool("force", false, "Force remove: remove tainted and outdated files") + flags.Bool("all", false, fmt.Sprintf("Remove all the %s", it.name)) + + return cmd +} + +func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { + run := func(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + hub, err := cwhub.GetHub() + if err != nil { + return err + } + + if all { + if err := hub.UpgradeConfig(it.name, "", force); err != nil { + return err + } + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one %s to upgrade or '--all'", it.singular) + } + + for _, name := range args { + if err := hub.UpgradeConfig(it.name, name, force); err != nil { + return err + } + } + + return nil + } + + return run +} + +func NewItemsUpgradeCmd(typeName string) *cobra.Command { + it := hubItemTypes[typeName] + + cmd := &cobra.Command{ + Use: coalesce.String(it.upgradeHelp.use, "upgrade [item]..."), + Short: coalesce.String(it.upgradeHelp.short, fmt.Sprintf("Upgrade given %s", it.oneOrMore)), + Long: coalesce.String(it.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", it.name)), + Example: it.upgradeHelp.example, + DisableAutoGenTag: true, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compInstalledItems(it.name, args, toComplete) + }, + RunE: itemsUpgradeRunner(it), + } + + flags := cmd.Flags() + flags.BoolP("all", "a", false, fmt.Sprintf("Upgrade all the %s", it.name)) + flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") + + return cmd +} + +func itemsInspectRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { + run := func(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + url, err := flags.GetString("url") + if err != nil { + return err + } + + if url != "" { + csConfig.Cscli.PrometheusUrl = url + } + + noMetrics, err := flags.GetBool("no-metrics") + if err != nil { + return err + } + + for _, name := range args { + if err = InspectItem(name, it.name, noMetrics); err != nil { + return err + } + } + + return nil + } + + return run +} + +func NewItemsInspectCmd(typeName string) *cobra.Command { + it := hubItemTypes[typeName] + + cmd := &cobra.Command{ + Use: coalesce.String(it.inspectHelp.use, "inspect [item]..."), + Short: coalesce.String(it.inspectHelp.short, fmt.Sprintf("Inspect given %s", it.oneOrMore)), + Long: coalesce.String(it.inspectHelp.long, fmt.Sprintf("Inspect the state of one or more %s", it.name)), + Example: it.inspectHelp.example, + Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compInstalledItems(it.name, args, toComplete) + }, + RunE: itemsInspectRunner(it), + } + + flags := cmd.Flags() + flags.StringP("url", "u", "", "Prometheus url") + flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") + + return cmd +} + +func itemsListRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { + run := func(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + if err = ListItems(color.Output, []string{it.name}, args, false, true, all); err != nil { + return err + } + + return nil + } + + return run +} + +func NewItemsListCmd(typeName string) *cobra.Command { + it := hubItemTypes[typeName] + + cmd := &cobra.Command{ + Use: coalesce.String(it.listHelp.use, "list [item... | -a]"), + Short: coalesce.String(it.listHelp.short, fmt.Sprintf("List %s", it.oneOrMore)), + Long: coalesce.String(it.listHelp.long, fmt.Sprintf("List of installed/available/specified %s", it.name)), + Example: it.listHelp.example, + DisableAutoGenTag: true, + RunE: itemsListRunner(it), + } + + flags := cmd.Flags() + flags.BoolP("all", "a", false, "List disabled items as well") + + return cmd +} diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index 1ddac4bf8..1d3aa56a6 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -234,10 +234,6 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall rootCmd.AddCommand(NewSimulationCmds()) rootCmd.AddCommand(NewBouncersCmd()) rootCmd.AddCommand(NewMachinesCmd()) - rootCmd.AddCommand(NewParsersCmd()) - rootCmd.AddCommand(NewScenariosCmd()) - rootCmd.AddCommand(NewCollectionsCmd()) - rootCmd.AddCommand(NewPostOverflowsCmd()) rootCmd.AddCommand(NewCapiCmd()) rootCmd.AddCommand(NewLapiCmd()) rootCmd.AddCommand(NewCompletionCmd()) @@ -246,6 +242,10 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall rootCmd.AddCommand(NewHubTestCmd()) rootCmd.AddCommand(NewNotificationsCmd()) rootCmd.AddCommand(NewSupportCmd()) + rootCmd.AddCommand(NewItemsCmd("collections")) + rootCmd.AddCommand(NewItemsCmd("parsers")) + rootCmd.AddCommand(NewItemsCmd("scenarios")) + rootCmd.AddCommand(NewItemsCmd("postoverflows")) if fflag.CscliSetup.IsEnabled() { rootCmd.AddCommand(NewSetupCmd()) diff --git a/cmd/crowdsec-cli/parsers.go b/cmd/crowdsec-cli/parsers.go deleted file mode 100644 index e0d294e01..000000000 --- a/cmd/crowdsec-cli/parsers.go +++ /dev/null @@ -1,320 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/fatih/color" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" -) - -func NewParsersCmd() *cobra.Command { - cmdParsers := &cobra.Command{ - Use: "parsers [parser]...", - Short: "Manage hub parsers", - Example: `cscli parsers list -a -cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs -cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs -cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs -cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs -`, - Args: cobra.MinimumNArgs(1), - Aliases: []string{"parser"}, - DisableAutoGenTag: true, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if _, err := require.Hub(csConfig); err != nil { - return err - } - - return nil - }, - PersistentPostRun: func(cmd *cobra.Command, args []string) { - if cmd.Name() == "inspect" || cmd.Name() == "list" { - return - } - log.Infof(ReloadMessage()) - }, - } - - cmdParsers.AddCommand(NewParsersInstallCmd()) - cmdParsers.AddCommand(NewParsersRemoveCmd()) - cmdParsers.AddCommand(NewParsersUpgradeCmd()) - cmdParsers.AddCommand(NewParsersInspectCmd()) - cmdParsers.AddCommand(NewParsersListCmd()) - - return cmdParsers -} - -func runParsersInstall(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - downloadOnly, err := flags.GetBool("download-only") - if err != nil { - return err - } - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - ignoreError, err := flags.GetBool("ignore") - if err != nil { - return err - } - - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - for _, name := range args { - t := hub.GetItem(cwhub.PARSERS, name) - if t == nil { - nearestItem, score := GetDistance(cwhub.PARSERS, name) - Suggest(cwhub.PARSERS, name, nearestItem.Name, score, ignoreError) - - continue - } - - if err := hub.InstallItem(name, cwhub.PARSERS, force, downloadOnly); err != nil { - if !ignoreError { - return fmt.Errorf("error while installing '%s': %w", name, err) - } - log.Errorf("Error while installing '%s': %s", name, err) - } - } - - return nil -} - -func NewParsersInstallCmd() *cobra.Command { - cmdParsersInstall := &cobra.Command{ - Use: "install ...", - Short: "Install given parser(s)", - Long: `Fetch and install one or more parsers from the hub`, - Example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compAllItems(cwhub.PARSERS, args, toComplete) - }, - RunE: runParsersInstall, - } - - flags := cmdParsersInstall.Flags() - flags.BoolP("download-only", "d", false, "Only download packages, don't enable") - flags.Bool("force", false, "Force install: overwrite tainted and outdated files") - flags.Bool("ignore", false, "Ignore errors when installing multiple parsers") - - return cmdParsersInstall -} - -func runParsersRemove(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - purge, err := flags.GetBool("purge") - if err != nil { - return err - } - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - if all { - err := hub.RemoveMany(cwhub.PARSERS, "", all, purge, force) - if err != nil { - return err - } - - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one parser to remove or '--all'") - } - - for _, name := range args { - err := hub.RemoveMany(cwhub.PARSERS, name, all, purge, force) - if err != nil { - return err - } - } - - return nil -} - -func NewParsersRemoveCmd() *cobra.Command { - cmdParsersRemove := &cobra.Command{ - Use: "remove ...", - Short: "Remove given parser(s)", - Long: `Remove one or more parsers`, - Example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, - Aliases: []string{"delete"}, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.PARSERS, args, toComplete) - }, - RunE: runParsersRemove, - } - - flags := cmdParsersRemove.Flags() - flags.Bool("purge", false, "Delete source file too") - flags.Bool("force", false, "Force remove: remove tainted and outdated files") - flags.Bool("all", false, "Remove all the parsers") - - return cmdParsersRemove -} - -func runParsersUpgrade(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - if all { - if err := hub.UpgradeConfig(cwhub.PARSERS, "", force); err != nil { - return err - } - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one parser to upgrade or '--all'") - } - - for _, name := range args { - if err := hub.UpgradeConfig(cwhub.PARSERS, name, force); err != nil { - return err - } - } - - return nil -} - -func NewParsersUpgradeCmd() *cobra.Command { - cmdParsersUpgrade := &cobra.Command{ - Use: "upgrade ...", - Short: "Upgrade given parser(s)", - Long: `Fetch and upgrade one or more parsers from the hub`, - Example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.PARSERS, args, toComplete) - }, - RunE: runParsersUpgrade, - } - - flags := cmdParsersUpgrade.Flags() - flags.BoolP("all", "a", false, "Upgrade all the parsers") - flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") - - return cmdParsersUpgrade -} - -func runParsersInspect(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - url, err := flags.GetString("url") - if err != nil { - return err - } - - if url != "" { - csConfig.Cscli.PrometheusUrl = url - } - - noMetrics, err := flags.GetBool("no-metrics") - if err != nil { - return err - } - - for _, name := range args { - if err = InspectItem(name, cwhub.PARSERS, noMetrics); err != nil { - return err - } - } - - return nil -} - -func NewParsersInspectCmd() *cobra.Command { - cmdParsersInspect := &cobra.Command{ - Use: "inspect ", - Short: "Inspect a parser", - Long: `Inspect a parser`, - Example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.PARSERS, args, toComplete) - }, - RunE: runParsersInspect, - } - - flags := cmdParsersInspect.Flags() - flags.StringP("url", "u", "", "Prometheus url") - flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") - - return cmdParsersInspect -} - -func runParsersList(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - if err = ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all); err != nil { - return err - } - - return nil -} - -func NewParsersListCmd() *cobra.Command { - cmdParsersList := &cobra.Command{ - Use: "list [parser... | -a]", - Short: "List parsers", - Long: `List of installed/available/specified parsers`, - Example: `cscli parsers list -cscli parsers list -a -cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, - DisableAutoGenTag: true, - RunE: runParsersList, - } - - flags := cmdParsersList.Flags() - flags.BoolP("all", "a", false, "List disabled items as well") - - return cmdParsersList -} diff --git a/cmd/crowdsec-cli/postoverflows.go b/cmd/crowdsec-cli/postoverflows.go deleted file mode 100644 index a73bf3da4..000000000 --- a/cmd/crowdsec-cli/postoverflows.go +++ /dev/null @@ -1,321 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/fatih/color" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" -) - -func NewPostOverflowsCmd() *cobra.Command { - cmdPostOverflows := &cobra.Command{ - Use: "postoverflows [postoverflow]...", - Short: "Manage hub postoverflows", - Example: `cscli postoverflows list -a -cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns -cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns -cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns -cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns -`, - Args: cobra.MinimumNArgs(1), - Aliases: []string{"postoverflow"}, - DisableAutoGenTag: true, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if _, err := require.Hub(csConfig); err != nil { - return err - } - - return nil - }, - PersistentPostRun: func(cmd *cobra.Command, args []string) { - if cmd.Name() == "inspect" || cmd.Name() == "list" { - return - } - log.Infof(ReloadMessage()) - }, - } - - cmdPostOverflows.AddCommand(NewPostOverflowsInstallCmd()) - cmdPostOverflows.AddCommand(NewPostOverflowsRemoveCmd()) - cmdPostOverflows.AddCommand(NewPostOverflowsUpgradeCmd()) - cmdPostOverflows.AddCommand(NewPostOverflowsInspectCmd()) - cmdPostOverflows.AddCommand(NewPostOverflowsListCmd()) - - return cmdPostOverflows -} - -func runPostOverflowsInstall(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - downloadOnly, err := flags.GetBool("download-only") - if err != nil { - return err - } - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - ignoreError, err := flags.GetBool("ignore") - if err != nil { - return err - } - - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - for _, name := range args { - t := hub.GetItem(cwhub.POSTOVERFLOWS, name) - if t == nil { - nearestItem, score := GetDistance(cwhub.POSTOVERFLOWS, name) - Suggest(cwhub.POSTOVERFLOWS, name, nearestItem.Name, score, ignoreError) - - continue - } - - if err := hub.InstallItem(name, cwhub.POSTOVERFLOWS, force, downloadOnly); err != nil { - if !ignoreError { - return fmt.Errorf("error while installing '%s': %w", name, err) - } - log.Errorf("Error while installing '%s': %s", name, err) - } - } - - return nil -} - -func NewPostOverflowsInstallCmd() *cobra.Command { - cmdPostOverflowsInstall := &cobra.Command{ - Use: "install ...", - Short: "Install given postoverflow(s)", - Long: `Fetch and install one or more postoverflows from the hub`, - Example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compAllItems(cwhub.POSTOVERFLOWS, args, toComplete) - }, - RunE: runPostOverflowsInstall, - } - - flags := cmdPostOverflowsInstall.Flags() - flags.BoolP("download-only", "d", false, "Only download packages, don't enable") - flags.Bool("force", false, "Force install: overwrite tainted and outdated files") - flags.Bool("ignore", false, "Ignore errors when installing multiple postoverflows") - - return cmdPostOverflowsInstall -} - -func runPostOverflowsRemove(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - purge, err := flags.GetBool("purge") - if err != nil { - return err - } - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - if all { - err := hub.RemoveMany(cwhub.POSTOVERFLOWS, "", all, purge, force) - if err != nil { - return err - } - - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one postoverflow to remove or '--all'") - } - - for _, name := range args { - err := hub.RemoveMany(cwhub.POSTOVERFLOWS, name, all, purge, force) - if err != nil { - return err - } - } - - return nil -} - -func NewPostOverflowsRemoveCmd() *cobra.Command { - cmdPostOverflowsRemove := &cobra.Command{ - Use: "remove ...", - Short: "Remove given postoverflow(s)", - Long: `remove one or more postoverflows from the hub`, - Example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`, - Aliases: []string{"delete"}, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.POSTOVERFLOWS, args, toComplete) - }, - RunE: runPostOverflowsRemove, - } - - flags := cmdPostOverflowsRemove.Flags() - flags.Bool("purge", false, "Delete source file too") - flags.Bool("force", false, "Force remove: remove tainted and outdated files") - flags.Bool("all", false, "Delete all the postoverflows") - - return cmdPostOverflowsRemove -} - -func runPostOverflowUpgrade(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - if all { - if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, "", force); err != nil { - return err - } - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one postoverflow to upgrade or '--all'") - } - - for _, name := range args { - if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, name, force); err != nil { - return err - } - } - - return nil -} - -func NewPostOverflowsUpgradeCmd() *cobra.Command { - cmdPostOverflowsUpgrade := &cobra.Command{ - Use: "upgrade ...", - Short: "Upgrade given postoverflow(s)", - Long: `Fetch and upgrade one or more postoverflows from the hub`, - Example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.POSTOVERFLOWS, args, toComplete) - }, - RunE: runPostOverflowUpgrade, - } - - flags := cmdPostOverflowsUpgrade.Flags() - flags.BoolP("all", "a", false, "Upgrade all the postoverflows") - flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") - - return cmdPostOverflowsUpgrade -} - -func runPostOverflowsInspect(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - url, err := flags.GetString("url") - if err != nil { - return err - } - - if url != "" { - csConfig.Cscli.PrometheusUrl = url - } - - noMetrics, err := flags.GetBool("no-metrics") - if err != nil { - return err - } - - for _, name := range args { - if err = InspectItem(name, cwhub.POSTOVERFLOWS, noMetrics); err != nil { - return err - } - } - - return nil -} - -func NewPostOverflowsInspectCmd() *cobra.Command { - cmdPostOverflowsInspect := &cobra.Command{ - Use: "inspect ", - Short: "Inspect a postoverflow", - Long: `Inspect a postoverflow`, - Example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.POSTOVERFLOWS, args, toComplete) - }, - RunE: runPostOverflowsInspect, - } - - flags := cmdPostOverflowsInspect.Flags() - - flags.StringP("url", "u", "", "Prometheus url") - flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") - - return cmdPostOverflowsInspect -} - -func runPostOverflowsList(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - if err = ListItems(color.Output, []string{cwhub.POSTOVERFLOWS}, args, false, true, all); err != nil { - return err - } - - return nil -} - -func NewPostOverflowsListCmd() *cobra.Command { - cmdPostOverflowsList := &cobra.Command{ - Use: "list [postoverflow]...", - Short: "List postoverflows", - Long: `List of installed/available/specified postoverflows`, - Example: `cscli postoverflows list -cscli postoverflows list -a -cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns`, - DisableAutoGenTag: true, - RunE: runPostOverflowsList, - } - - flags := cmdPostOverflowsList.Flags() - flags.BoolP("all", "a", false, "List disabled items as well") - - return cmdPostOverflowsList -} diff --git a/cmd/crowdsec-cli/scenarios.go b/cmd/crowdsec-cli/scenarios.go deleted file mode 100644 index 23df3f5ff..000000000 --- a/cmd/crowdsec-cli/scenarios.go +++ /dev/null @@ -1,320 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/fatih/color" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" -) - -func NewScenariosCmd() *cobra.Command { - cmdScenarios := &cobra.Command{ - Use: "scenarios [scenario]...", - Short: "Manage hub scenarios", - Example: `cscli scenarios list -a -cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing -cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing -cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing -cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing -`, - Args: cobra.MinimumNArgs(1), - Aliases: []string{"scenario"}, - DisableAutoGenTag: true, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if _, err := require.Hub(csConfig); err != nil { - return err - } - - return nil - }, - PersistentPostRun: func(cmd *cobra.Command, args []string) { - if cmd.Name() == "inspect" || cmd.Name() == "list" { - return - } - log.Infof(ReloadMessage()) - }, - } - - cmdScenarios.AddCommand(NewCmdScenariosInstall()) - cmdScenarios.AddCommand(NewCmdScenariosRemove()) - cmdScenarios.AddCommand(NewCmdScenariosUpgrade()) - cmdScenarios.AddCommand(NewCmdScenariosInspect()) - cmdScenarios.AddCommand(NewCmdScenariosList()) - - return cmdScenarios -} - -func runScenariosInstall(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - downloadOnly, err := flags.GetBool("download-only") - if err != nil { - return err - } - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - ignoreError, err := flags.GetBool("ignore") - if err != nil { - return err - } - - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - for _, name := range args { - t := hub.GetItem(cwhub.SCENARIOS, name) - if t == nil { - nearestItem, score := GetDistance(cwhub.SCENARIOS, name) - Suggest(cwhub.SCENARIOS, name, nearestItem.Name, score, ignoreError) - - continue - } - - if err := hub.InstallItem(name, cwhub.SCENARIOS, force, downloadOnly); err != nil { - if !ignoreError { - return fmt.Errorf("error while installing '%s': %w", name, err) - } - log.Errorf("Error while installing '%s': %s", name, err) - } - } - - return nil -} - -func NewCmdScenariosInstall() *cobra.Command { - cmdScenariosInstall := &cobra.Command{ - Use: "install ...", - Short: "Install given scenario(s)", - Long: `Fetch and install one or more scenarios from the hub`, - Example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compAllItems(cwhub.SCENARIOS, args, toComplete) - }, - RunE: runScenariosInstall, - } - - flags := cmdScenariosInstall.Flags() - flags.BoolP("download-only", "d", false, "Only download packages, don't enable") - flags.Bool("force", false, "Force install: overwrite tainted and outdated files") - flags.Bool("ignore", false, "Ignore errors when installing multiple scenarios") - - return cmdScenariosInstall -} - -func runScenariosRemove(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - purge, err := flags.GetBool("purge") - if err != nil { - return err - } - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - if all { - err := hub.RemoveMany(cwhub.SCENARIOS, "", all, purge, force) - if err != nil { - return err - } - - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one scenario to remove or '--all'") - } - - for _, name := range args { - err := hub.RemoveMany(cwhub.SCENARIOS, name, all, purge, force) - if err != nil { - return err - } - } - - return nil -} - -func NewCmdScenariosRemove() *cobra.Command { - cmdScenariosRemove := &cobra.Command{ - Use: "remove ...", - Short: "Remove given scenario(s)", - Long: `remove one or more scenarios`, - Example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`, - Aliases: []string{"delete"}, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.SCENARIOS, args, toComplete) - }, - RunE: runScenariosRemove, - } - - flags := cmdScenariosRemove.Flags() - flags.Bool("purge", false, "Delete source file too") - flags.Bool("force", false, "Force remove: remove tainted and outdated files") - flags.Bool("all", false, "Remove all the scenarios") - - return cmdScenariosRemove -} - -func runScenariosUpgrade(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - if all { - if err := hub.UpgradeConfig(cwhub.SCENARIOS, "", force); err != nil { - return err - } - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one scenario to upgrade or '--all'") - } - - for _, name := range args { - if err := hub.UpgradeConfig(cwhub.SCENARIOS, name, force); err != nil { - return err - } - } - - return nil -} - -func NewCmdScenariosUpgrade() *cobra.Command { - cmdScenariosUpgrade := &cobra.Command{ - Use: "upgrade ...", - Short: "Upgrade given scenario(s)", - Long: `Fetch and upgrade one or more scenarios from the hub`, - Example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.SCENARIOS, args, toComplete) - }, - RunE: runScenariosUpgrade, - } - - flags := cmdScenariosUpgrade.Flags() - flags.BoolP("all", "a", false, "Upgrade all the scenarios") - flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") - - return cmdScenariosUpgrade -} - -func runScenariosInspect(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - url, err := flags.GetString("url") - if err != nil { - return err - } - - if url != "" { - csConfig.Cscli.PrometheusUrl = url - } - - noMetrics, err := flags.GetBool("no-metrics") - if err != nil { - return err - } - - for _, name := range args { - if err = InspectItem(name, cwhub.SCENARIOS, noMetrics); err != nil { - return err - } - } - - return nil -} - -func NewCmdScenariosInspect() *cobra.Command { - cmdScenariosInspect := &cobra.Command{ - Use: "inspect ", - Short: "Inspect a scenario", - Long: `Inspect a scenario`, - Example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.SCENARIOS, args, toComplete) - }, - RunE: runScenariosInspect, - } - - flags := cmdScenariosInspect.Flags() - flags.StringP("url", "u", "", "Prometheus url") - flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") - - return cmdScenariosInspect -} - -func runScenariosList(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - if err = ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all); err != nil { - return err - } - - return nil -} - -func NewCmdScenariosList() *cobra.Command { - cmdScenariosList := &cobra.Command{ - Use: "list [scenario]...", - Short: "List scenarios", - Long: `List of installed/available/specified scenarios`, - Example: `cscli scenarios list -cscli scenarios list -a -cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing`, - DisableAutoGenTag: true, - RunE: runScenariosList, - } - - flags := cmdScenariosList.Flags() - flags.BoolP("all", "a", false, "List disabled items as well") - - return cmdScenariosList -} diff --git a/go.mod b/go.mod index 5d641b084..15b3ed07a 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/c-robinson/iplib v1.0.3 github.com/cespare/xxhash/v2 v2.2.0 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 - github.com/crowdsecurity/go-cs-lib v0.0.4 + github.com/crowdsecurity/go-cs-lib v0.0.5 github.com/crowdsecurity/grokky v0.2.1 github.com/crowdsecurity/machineid v1.0.2 github.com/davecgh/go-spew v1.1.1 diff --git a/go.sum b/go.sum index 5c9af48bd..82d02546a 100644 --- a/go.sum +++ b/go.sum @@ -137,8 +137,8 @@ github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= -github.com/crowdsecurity/go-cs-lib v0.0.4 h1:mH3iqz8H8iH9YpldqCdojyKHy9z3JDhas/k6I8M0ims= -github.com/crowdsecurity/go-cs-lib v0.0.4/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k= +github.com/crowdsecurity/go-cs-lib v0.0.5 h1:eVLW+BRj3ZYn0xt5/xmgzfbbB8EBo32gM4+WpQQk2e8= +github.com/crowdsecurity/go-cs-lib v0.0.5/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k= github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4= github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM= github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc= diff --git a/pkg/cwhub/branch.go b/pkg/cwhub/branch.go new file mode 100644 index 000000000..e25a6becf --- /dev/null +++ b/pkg/cwhub/branch.go @@ -0,0 +1,59 @@ +package cwhub + +// Set the appropriate hub branch according to config settings and crowdsec version + +import ( + log "github.com/sirupsen/logrus" + "golang.org/x/mod/semver" + + "github.com/crowdsecurity/crowdsec/pkg/cwversion" +) + +// chooseHubBranch returns the branch name to use for the hub +// It can be "master" or the branch corresponding to the current crowdsec version +func chooseHubBranch() string { + latest, err := cwversion.Latest() + if err != nil { + log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err) + return "master" + } + + csVersion := cwversion.VersionStrip() + if csVersion == latest { + log.Debugf("current version is equal to latest (%s)", csVersion) + return "master" + } + + // if current version is greater than the latest we are in pre-release + if semver.Compare(csVersion, latest) == 1 { + log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion) + return "master" + } + + if csVersion == "" { + log.Warning("Crowdsec version is not set, using master branch for the hub") + return "master" + } + + log.Warnf("Crowdsec is not the latest version. "+ + "Current version is '%s' and the latest stable version is '%s'. Please update it!", + csVersion, latest) + + log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+ + "added to Crowdsec Hub after CrowdSec %s", latest) + + return csVersion +} + +// SetHubBranch sets the package variable that points to the hub branch. +func SetHubBranch() { + // a branch is already set, or specified from the flags + if HubBranch != "" { + return + } + + // use the branch corresponding to the crowdsec version + HubBranch = chooseHubBranch() + + log.Debugf("Using branch '%s' for the hub", HubBranch) +} diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index 7ae805492..d1f155813 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -5,14 +5,7 @@ package cwhub import ( - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/enescakir/emoji" - "github.com/pkg/errors" - "golang.org/x/mod/semver" + "errors" ) var ( @@ -21,230 +14,3 @@ var ( RawFileURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s" HubBranch = "master" ) - -// ItemVersion is used to detect the version of a given item -// by comparing the hash of each version to the local file. -// If the item does not match any known version, it is considered tainted. -type ItemVersion struct { - Digest string `json:"digest,omitempty"` // meow - Deprecated bool `json:"deprecated,omitempty"` // XXX: do we keep this? -} - -// Item represents an object managed in the hub. It can be a parser, scenario, collection.. -type Item struct { - // descriptive info - Type string `json:"type,omitempty" yaml:"type,omitempty"` // parser|postoverflows|scenario|collection(|enrich) - Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-... - Name string `json:"name,omitempty"` // as seen in .index.json, usually "author/name" - FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml - Description string `json:"description,omitempty" yaml:"description,omitempty"` // as seen in .index.json - Author string `json:"author,omitempty"` // as seen in .index.json - References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .index.json - BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any - - // remote (hub) info - RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // the path relative to (git | hub API) ie. /parsers/stage/author/file.yaml - Version string `json:"version,omitempty"` // the last version - Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // the list of existing versions - - // local (deployed) info - LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR} - LocalVersion string `json:"local_version,omitempty"` - LocalHash string `json:"local_hash,omitempty"` // the local meow - Installed bool `json:"installed,omitempty"` - Downloaded bool `json:"downloaded,omitempty"` - UpToDate bool `json:"up_to_date,omitempty"` - Tainted bool `json:"tainted,omitempty"` // has it been locally modified - Local bool `json:"local,omitempty"` // if it's a non versioned control one - - // if it's a collection, it can have sub items - Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"` - PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"` - Scenarios []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"` - Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"` -} - -// Status returns the status of the item as a string and an emoji -// ie. "enabled,update-available" and emoji.Warning -func (i *Item) Status() (string, emoji.Emoji) { - status := "disabled" - ok := false - - if i.Installed { - ok = true - status = "enabled" - } - - managed := true - if i.Local { - managed = false - status += ",local" - } - - warning := false - if i.Tainted { - warning = true - status += ",tainted" - } else if !i.UpToDate && !i.Local { - warning = true - status += ",update-available" - } - - emo := emoji.QuestionMark - - switch { - case !managed: - emo = emoji.House - case !i.Installed: - emo = emoji.Prohibited - case warning: - emo = emoji.Warning - case ok: - emo = emoji.CheckMark - } - - return status, emo -} - -// versionStatus: semver requires 'v' prefix -func (i *Item) versionStatus() int { - return semver.Compare("v"+i.Version, "v"+i.LocalVersion) -} - -// validPath returns true if the (relative) path is allowed for the item -// dirNmae: the directory name (ie. crowdsecurity) -// fileName: the filename (ie. apache2-logs.yaml) -func (i *Item) validPath(dirName, fileName string) bool { - return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml") -} - -// GetItemMap returns the map of items for a given type -func (h *Hub) GetItemMap(itemType string) map[string]Item { - m, ok := h.Items[itemType] - if !ok { - return nil - } - - return m -} - -// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary -// XXX: only used by leakybucket manager -func itemKey(itemPath string) (string, error) { - f, err := os.Lstat(itemPath) - if err != nil { - return "", fmt.Errorf("while performing lstat on %s: %w", itemPath, err) - } - - if f.Mode()&os.ModeSymlink == 0 { - // it's not a symlink, so the filename itsef should be the key - return filepath.Base(itemPath), nil - } - - // resolve the symlink to hub file - pathInHub, err := os.Readlink(itemPath) - if err != nil { - return "", fmt.Errorf("while reading symlink of %s: %w", itemPath, err) - } - - author := filepath.Base(filepath.Dir(pathInHub)) - - fname := filepath.Base(pathInHub) - fname = strings.TrimSuffix(fname, ".yaml") - fname = strings.TrimSuffix(fname, ".yml") - - return fmt.Sprintf("%s/%s", author, fname), nil -} - -// GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item. -func (h *Hub) GetItemByPath(itemType string, itemPath string) (*Item, error) { - itemKey, err := itemKey(itemPath) - if err != nil { - return nil, err - } - - m := h.GetItemMap(itemType) - if m == nil { - return nil, fmt.Errorf("item type %s doesn't exist", itemType) - } - - v, ok := m[itemKey] - if !ok { - return nil, fmt.Errorf("%s not found in %s", itemKey, itemType) - } - - return &v, nil -} - -// GetItem returns the item from hub based on its type and full name (author/name) -func (h *Hub) GetItem(itemType string, itemName string) *Item { - m, ok := h.GetItemMap(itemType)[itemName] - if !ok { - return nil - } - - return &m -} - -// GetItemNames returns the list of item (full) names for a given type -// ie. for parsers: crowdsecurity/apache2 crowdsecurity/nginx -// The names can be used to retrieve the item with GetItem() -func (h *Hub) GetItemNames(itemType string) []string { - m := h.GetItemMap(itemType) - if m == nil { - return nil - } - - names := make([]string, 0, len(m)) - for k := range m { - names = append(names, k) - } - - return names -} - -// AddItem adds an item to the hub index -func (h *Hub) AddItem(itemType string, item Item) error { - for _, itype := range ItemTypes { - if itype == itemType { - h.Items[itemType][item.Name] = item - return nil - } - } - - return fmt.Errorf("ItemType %s is unknown", itemType) -} - -// GetInstalledItems returns the list of installed items -func (h *Hub) GetInstalledItems(itemType string) ([]Item, error) { - items, ok := h.Items[itemType] - if !ok { - return nil, fmt.Errorf("no %s in hubIdx", itemType) - } - - retItems := make([]Item, 0) - - for _, item := range items { - if item.Installed { - retItems = append(retItems, item) - } - } - - return retItems, nil -} - -// GetInstalledItemsAsString returns the names of the installed items -func (h *Hub) GetInstalledItemsAsString(itemType string) ([]string, error) { - items, err := h.GetInstalledItems(itemType) - if err != nil { - return nil, err - } - - retStr := make([]string, len(items)) - - for i, it := range items { - retStr[i] = it.Name - } - - return retStr, nil -} diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index 216a0df4b..b51c3aa6e 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -9,11 +9,8 @@ import ( "testing" log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/crowdsecurity/go-cs-lib/cstest" - "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) @@ -28,81 +25,6 @@ import ( var responseByPath map[string]string -func TestItemStatus(t *testing.T) { - hub := envSetup(t) - - // get existing map - x := hub.GetItemMap(COLLECTIONS) - require.NotEmpty(t, x) - - // Get item : good and bad - for k := range x { - item := hub.GetItem(COLLECTIONS, k) - require.NotNil(t, item) - - item.Installed = true - item.UpToDate = false - item.Local = false - item.Tainted = false - - txt, _ := item.Status() - require.Equal(t, "enabled,update-available", txt) - - item.Installed = false - item.UpToDate = false - item.Local = true - item.Tainted = false - - txt, _ = item.Status() - require.Equal(t, "disabled,local", txt) - } - - err := DisplaySummary() - require.NoError(t, err) -} - -func TestGetters(t *testing.T) { - hub := envSetup(t) - - // get non existing map - empty := hub.GetItemMap("ratata") - require.Nil(t, empty) - - // get existing map - x := hub.GetItemMap(COLLECTIONS) - require.NotEmpty(t, x) - - // Get item : good and bad - for k := range x { - empty := hub.GetItem(COLLECTIONS, k+"nope") - require.Nil(t, empty) - - item := hub.GetItem(COLLECTIONS, k) - require.NotNil(t, item) - - // Add item and get it - item.Name += "nope" - err := hub.AddItem(COLLECTIONS, *item) - require.NoError(t, err) - - newitem := hub.GetItem(COLLECTIONS, item.Name) - require.NotNil(t, newitem) - - err = hub.AddItem("ratata", *item) - cstest.RequireErrorContains(t, err, "ItemType ratata is unknown") - } -} - -func TestIndexDownload(t *testing.T) { - hub := envSetup(t) - - _, err := InitHubUpdate(hub.cfg) - require.NoError(t, err, "failed to download index") - - _, err = GetHub() - require.NoError(t, err, "failed to load hub index") -} - // testHub initializes a temporary hub with an empty json file, optionally updating it func testHub(t *testing.T, update bool) *Hub { tmpDir, err := os.MkdirTemp("", "testhub") @@ -115,13 +37,13 @@ func testHub(t *testing.T, update bool) *Hub { InstallDataDir: filepath.Join(tmpDir, "installed-data"), } - err = os.MkdirAll(hubCfg.HubDir, 0700) + err = os.MkdirAll(hubCfg.HubDir, 0o700) require.NoError(t, err) - err = os.MkdirAll(hubCfg.InstallDir, 0700) + err = os.MkdirAll(hubCfg.InstallDir, 0o700) require.NoError(t, err) - err = os.MkdirAll(hubCfg.InstallDataDir, 0700) + err = os.MkdirAll(hubCfg.InstallDataDir, 0o700) require.NoError(t, err) index, err := os.Create(hubCfg.HubIndexFile) @@ -148,8 +70,9 @@ func testHub(t *testing.T, update bool) *Hub { return hub } +// envSetup initializes the temporary hub and mocks the http client func envSetup(t *testing.T) *Hub { - resetResponseByPath() + setResponseByPath() log.SetLevel(log.DebugLevel) defaultTransport := http.DefaultClient.Transport @@ -163,151 +86,9 @@ func envSetup(t *testing.T) *Hub { hub := testHub(t, true) - // if err := os.RemoveAll(cfg.Hub.InstallDir); err != nil { - // log.Fatalf("failed to remove %s : %s", cfg.Hub.InstallDir, err) - // } - // if err := os.MkdirAll(cfg.Hub.InstallDir, 0700); err != nil { - // log.Fatalf("failed to mkdir %s : %s", cfg.Hub.InstallDir, err) - // } return hub } -func testInstallItem(hub *Hub, t *testing.T, item Item) { - // Install the parser - - err := hub.DownloadLatest(&item, false, false) - require.NoError(t, err, "failed to download %s", item.Name) - - _, err = hub.LocalSync() - require.NoError(t, err, "failed to run localSync") - - assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) - - err = hub.EnableItem(&item) - require.NoError(t, err, "failed to enable %s", item.Name) - - _, err = hub.LocalSync() - require.NoError(t, err, "failed to run localSync") - - assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) -} - -func testTaintItem(hub *Hub, t *testing.T, item Item) { - assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) - - f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600) - require.NoError(t, err, "failed to open %s (%s)", item.LocalPath, item.Name) - - defer f.Close() - - _, err = f.WriteString("tainted") - require.NoError(t, err, "failed to write to %s (%s)", item.LocalPath, item.Name) - - // Local sync and check status - _, err = hub.LocalSync() - require.NoError(t, err, "failed to run localSync") - - assert.True(t, hub.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name) -} - -func testUpdateItem(hub *Hub, t *testing.T, item Item) { - assert.False(t, hub.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name) - - // Update it + check status - err := hub.DownloadLatest(&item, true, true) - require.NoError(t, err, "failed to update %s", item.Name) - - // Local sync and check status - _, err = hub.LocalSync() - require.NoError(t, err, "failed to run localSync") - - assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) -} - -func testDisableItem(hub *Hub, t *testing.T, item Item) { - assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) - - // Remove - err := hub.DisableItem(&item, false, false) - require.NoError(t, err, "failed to disable %s", item.Name) - - // Local sync and check status - warns, err := hub.LocalSync() - require.NoError(t, err, "failed to run localSync") - require.Empty(t, warns, "unexpected warnings : %+v", warns) - - assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) - assert.True(t, hub.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name) - - // Purge - err = hub.DisableItem(&item, true, false) - require.NoError(t, err, "failed to purge %s", item.Name) - - // Local sync and check status - warns, err = hub.LocalSync() - require.NoError(t, err, "failed to run localSync") - require.Empty(t, warns, "unexpected warnings : %+v", warns) - - assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name) -} - -func TestInstallParser(t *testing.T) { - /* - - install a random parser - - check its status - - taint it - - check its status - - force update it - - check its status - - remove it - */ - hub := envSetup(t) - - // map iteration is random by itself - for _, it := range hub.Items[PARSERS] { - testInstallItem(hub, t, it) - it = hub.Items[PARSERS][it.Name] - testTaintItem(hub, t, it) - it = hub.Items[PARSERS][it.Name] - testUpdateItem(hub, t, it) - it = hub.Items[PARSERS][it.Name] - testDisableItem(hub, t, it) - it = hub.Items[PARSERS][it.Name] - - break - } -} - -func TestInstallCollection(t *testing.T) { - /* - - install a random parser - - check its status - - taint it - - check its status - - force update it - - check its status - - remove it - */ - hub := envSetup(t) - - // map iteration is random by itself - for _, it := range hub.Items[COLLECTIONS] { - testInstallItem(hub, t, it) - it = hub.Items[COLLECTIONS][it.Name] - testTaintItem(hub, t, it) - it = hub.Items[COLLECTIONS][it.Name] - testUpdateItem(hub, t, it) - it = hub.Items[COLLECTIONS][it.Name] - testDisableItem(hub, t, it) - break - } -} - type mockTransport struct{} func newMockTransport() http.RoundTripper { @@ -352,7 +133,7 @@ func fileToStringX(path string) string { return strings.ReplaceAll(string(data), "\r\n", "\n") } -func resetResponseByPath() { +func setResponseByPath() { responseByPath = map[string]string{ "/master/parsers/s01-parse/crowdsecurity/foobar_parser.yaml": fileToStringX("./testdata/foobar_parser.yaml"), "/master/parsers/s01-parse/crowdsecurity/foobar_subparser.yaml": fileToStringX("./testdata/foobar_parser.yaml"), diff --git a/pkg/cwhub/dataset.go b/pkg/cwhub/dataset.go index 2255d40a7..e031fde3a 100644 --- a/pkg/cwhub/dataset.go +++ b/pkg/cwhub/dataset.go @@ -1,6 +1,7 @@ package cwhub import ( + "errors" "fmt" "io" "net/http" @@ -8,6 +9,7 @@ import ( "path/filepath" log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -39,7 +41,7 @@ func downloadFile(url string, destPath string) error { return fmt.Errorf("download response 'HTTP %d' : %s", resp.StatusCode, string(body)) } - file, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + file, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644) if err != nil { return err } @@ -70,3 +72,40 @@ func GetData(data []*types.DataSource, dataDir string) error { return nil } + +// downloadData downloads the data files for an item +func downloadData(dataFolder string, force bool, reader io.Reader) error { + var err error + + dec := yaml.NewDecoder(reader) + + for { + data := &DataSet{} + + err = dec.Decode(data) + if err != nil { + if errors.Is(err, io.EOF) { + break + } + + return fmt.Errorf("while reading file: %w", err) + } + + download := false + + for _, dataS := range data.Data { + if _, err = os.Stat(filepath.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) { + download = true + } + } + + if download || force { + err = GetData(data.Data, dataFolder) + if err != nil { + return fmt.Errorf("while getting data: %w", err) + } + } + } + + return nil +} diff --git a/pkg/cwhub/dataset_test.go b/pkg/cwhub/dataset_test.go index 40f6ba847..a072fbb1c 100644 --- a/pkg/cwhub/dataset_test.go +++ b/pkg/cwhub/dataset_test.go @@ -14,12 +14,14 @@ func TestDownloadFile(t *testing.T) { httpmock.Activate() defer httpmock.DeactivateAndReset() + //OK httpmock.RegisterResponder( "GET", "https://example.com/xx", httpmock.NewStringResponder(200, "example content oneoneone"), ) + httpmock.RegisterResponder( "GET", "https://example.com/x", @@ -28,15 +30,19 @@ func TestDownloadFile(t *testing.T) { err := downloadFile("https://example.com/xx", examplePath) assert.NoError(t, err) + content, err := os.ReadFile(examplePath) assert.Equal(t, "example content oneoneone", string(content)) assert.NoError(t, err) + //bad uri err = downloadFile("https://zz.com", examplePath) assert.Error(t, err) + //404 err = downloadFile("https://example.com/x", examplePath) assert.Error(t, err) + //bad target err = downloadFile("https://example.com/xx", "") assert.Error(t, err) diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go deleted file mode 100644 index 0f4a8fb7a..000000000 --- a/pkg/cwhub/download.go +++ /dev/null @@ -1,336 +0,0 @@ -package cwhub - -import ( - "bytes" - "crypto/sha256" - "errors" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "strings" - - log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" - - "github.com/crowdsecurity/crowdsec/pkg/csconfig" -) - -var ErrIndexNotFound = fmt.Errorf("index not found") - -// InitHubUpdate is like InitHub but downloads and updates the index instead of reading from the disk -// It is used to inizialize the hub when there is no index file yet -func InitHubUpdate(cfg *csconfig.HubCfg) (*Hub, error) { - if cfg == nil { - return nil, fmt.Errorf("no configuration found for hub") - } - - bidx, err := DownloadHubIdx(cfg.HubIndexFile) - if err != nil { - return nil, fmt.Errorf("failed to download index: %w", err) - } - - ret, err := ParseIndex(bidx) - if err != nil { - if !errors.Is(err, ErrMissingReference) { - return nil, fmt.Errorf("failed to read index: %w", err) - } - } - - theHub = &Hub{ - Items: ret, - cfg: cfg, - } - - if _, err := theHub.LocalSync(); err != nil { - return nil, fmt.Errorf("failed to sync: %w", err) - } - - return theHub, nil -} - -// DownloadHubIdx downloads the latest version of the index and returns the content -func DownloadHubIdx(indexPath string) ([]byte, error) { - log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile)) - - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil) - if err != nil { - return nil, fmt.Errorf("failed to build request for hub index: %w", err) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, fmt.Errorf("failed http request for hub index: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - if resp.StatusCode == http.StatusNotFound { - return nil, ErrIndexNotFound - } - - return nil, fmt.Errorf("bad http code %d while requesting %s", resp.StatusCode, req.URL.String()) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read request answer for hub index: %w", err) - } - - oldContent, err := os.ReadFile(indexPath) - if err != nil { - if !os.IsNotExist(err) { - log.Warningf("failed to read hub index: %s", err) - } - } else if bytes.Equal(body, oldContent) { - log.Info("hub index is up to date") - // write it anyway, can't hurt - } - - file, err := os.OpenFile(indexPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - - if err != nil { - return nil, fmt.Errorf("while opening hub index file: %w", err) - } - defer file.Close() - - wsize, err := file.Write(body) - if err != nil { - return nil, fmt.Errorf("while writing hub index file: %w", err) - } - - log.Infof("Wrote new %d bytes index to %s", wsize, indexPath) - - return body, nil -} - -// DownloadLatest will download the latest version of Item to the tdir directory -func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) error { - var err error - - log.Debugf("Downloading %s %s", target.Type, target.Name) - - if target.Type != COLLECTIONS { - if !target.Installed && updateOnly && target.Downloaded { - log.Debugf("skipping upgrade of %s : not installed", target.Name) - return nil - } - - return h.DownloadItem(target, overwrite) - } - - // collection - var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} - for idx, ptr := range tmp { - ptrtype := ItemTypes[idx] - for _, p := range ptr { - val, ok := h.Items[ptrtype][p] - if !ok { - return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name) - } - - if !val.Installed && updateOnly && val.Downloaded { - log.Debugf("skipping upgrade of %s : not installed", target.Name) - continue - } - - log.Debugf("Download %s sub-item : %s %s (%t -> %t)", target.Name, ptrtype, p, target.Installed, updateOnly) - //recurse as it's a collection - if ptrtype == COLLECTIONS { - log.Tracef("collection, recurse") - - err = h.DownloadLatest(&val, overwrite, updateOnly) - if err != nil { - return fmt.Errorf("while downloading %s: %w", val.Name, err) - } - } - - downloaded := val.Downloaded - - err = h.DownloadItem(&val, overwrite) - if err != nil { - return fmt.Errorf("while downloading %s: %w", val.Name, err) - } - - // We need to enable an item when it has been added to a collection since latest release of the collection. - // We check if val.Downloaded is false because maybe the item has been disabled by the user. - if !val.Installed && !downloaded { - if err = h.EnableItem(&val); err != nil { - return fmt.Errorf("enabling '%s': %w", val.Name, err) - } - } - - h.Items[ptrtype][p] = val - } - } - - err = h.DownloadItem(target, overwrite) - if err != nil { - return fmt.Errorf("failed to download item: %w", err) - } - - return nil -} - -func (h *Hub) DownloadItem(target *Item, overwrite bool) error { - tdir := h.cfg.HubDir - - // if user didn't --force, don't overwrite local, tainted, up-to-date files - if !overwrite { - if target.Tainted { - log.Debugf("%s : tainted, not updated", target.Name) - return nil - } - - if target.UpToDate { - // We still have to check if data files are present - log.Debugf("%s : up-to-date, not updated", target.Name) - } - } - - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil) - if err != nil { - return fmt.Errorf("while downloading %s: %w", req.URL.String(), err) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return fmt.Errorf("while downloading %s: %w", req.URL.String(), err) - } - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String()) - } - - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("while reading %s: %w", req.URL.String(), err) - } - - hash := sha256.New() - if _, err = hash.Write(body); err != nil { - return fmt.Errorf("while hashing %s: %w", target.Name, err) - } - - meow := fmt.Sprintf("%x", hash.Sum(nil)) - if meow != target.Versions[target.Version].Digest { - log.Errorf("Downloaded version doesn't match index, please 'hub update'") - log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest) - - return fmt.Errorf("invalid download hash for %s", target.Name) - } - - //all good, install - //check if parent dir exists - tmpdirs := strings.Split(tdir+"/"+target.RemotePath, "/") - parentDir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/") - - // ensure that target file is within target dir - finalPath, err := filepath.Abs(tdir + "/" + target.RemotePath) - if err != nil { - return fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+target.RemotePath, err) - } - - if !strings.HasPrefix(finalPath, tdir) { - return fmt.Errorf("path %s escapes %s, abort", target.RemotePath, tdir) - } - - // check dir - if _, err = os.Stat(parentDir); os.IsNotExist(err) { - log.Debugf("%s doesn't exist, create", parentDir) - - if err = os.MkdirAll(parentDir, os.ModePerm); err != nil { - return fmt.Errorf("while creating parent directories: %w", err) - } - } - - // check actual file - if _, err = os.Stat(finalPath); !os.IsNotExist(err) { - log.Warningf("%s : overwrite", target.Name) - log.Debugf("target: %s/%s", tdir, target.RemotePath) - } else { - log.Infof("%s : OK", target.Name) - } - - f, err := os.OpenFile(tdir+"/"+target.RemotePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - if err != nil { - return fmt.Errorf("while opening file: %w", err) - } - - defer f.Close() - - _, err = f.Write(body) - if err != nil { - return fmt.Errorf("while writing file: %w", err) - } - - target.Downloaded = true - target.Tainted = false - target.UpToDate = true - - if err = downloadData(h.cfg.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil { - return fmt.Errorf("while downloading data for %s: %w", target.FileName, err) - } - - h.Items[target.Type][target.Name] = *target - - return nil -} - -// DownloadDataIfNeeded downloads the data files for an item -func (h *Hub) DownloadDataIfNeeded(target Item, force bool) error { - itemFilePath := fmt.Sprintf("%s/%s/%s/%s", h.cfg.InstallDir, target.Type, target.Stage, target.FileName) - - itemFile, err := os.Open(itemFilePath) - if err != nil { - return fmt.Errorf("while opening %s: %w", itemFilePath, err) - } - - defer itemFile.Close() - - if err = downloadData(h.cfg.InstallDataDir, force, itemFile); err != nil { - return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err) - } - - return nil -} - -// downloadData downloads the data files for an item -func downloadData(dataFolder string, force bool, reader io.Reader) error { - var err error - - dec := yaml.NewDecoder(reader) - - for { - data := &DataSet{} - - err = dec.Decode(data) - if err != nil { - if errors.Is(err, io.EOF) { - break - } - - return fmt.Errorf("while reading file: %w", err) - } - - download := false - - for _, dataS := range data.Data { - if _, err = os.Stat(filepath.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) { - download = true - } - } - - if download || force { - err = GetData(data.Data, dataFolder) - if err != nil { - return fmt.Errorf("while getting data: %w", err) - } - } - } - - return nil -} diff --git a/pkg/cwhub/download_test.go b/pkg/cwhub/download_test.go deleted file mode 100644 index ed0763081..000000000 --- a/pkg/cwhub/download_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package cwhub - -import ( - "fmt" - "os" - "strings" - "testing" - - log "github.com/sirupsen/logrus" -) - -func TestDownloadHubIdx(t *testing.T) { - back := RawFileURLTemplate - // bad url template - fmt.Println("Test 'bad URL'") - - tmpIndex, err := os.CreateTemp("", "index.json") - if err != nil { - t.Fatalf("failed to create temp file : %s", err) - } - - t.Cleanup(func() { - os.Remove(tmpIndex.Name()) - }) - - RawFileURLTemplate = "x" - - ret, err := DownloadHubIdx(tmpIndex.Name()) - if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") { - log.Errorf("unexpected error %s", err) - } - - fmt.Printf("->%+v", ret) - - // bad domain - fmt.Println("Test 'bad domain'") - - RawFileURLTemplate = "https://baddomain/%s/%s" - - ret, err = DownloadHubIdx(tmpIndex.Name()) - if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") { - log.Errorf("unexpected error %s", err) - } - - fmt.Printf("->%+v", ret) - - // bad target path - fmt.Println("Test 'bad target path'") - - RawFileURLTemplate = back - - ret, err = DownloadHubIdx("/does/not/exist/index.json") - if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") { - log.Errorf("unexpected error %s", err) - } - - RawFileURLTemplate = back - - fmt.Printf("->%+v", ret) -} diff --git a/pkg/cwhub/install.go b/pkg/cwhub/enable.go similarity index 78% rename from pkg/cwhub/install.go rename to pkg/cwhub/enable.go index 71fb46b1b..4aac75927 100644 --- a/pkg/cwhub/install.go +++ b/pkg/cwhub/enable.go @@ -1,5 +1,8 @@ package cwhub +// Enable/disable items already installed (no downloading here) +// This file is not named install.go to avoid confusion with the functions in helpers.go + import ( "fmt" "os" @@ -8,6 +11,81 @@ import ( log "github.com/sirupsen/logrus" ) +// creates symlink between actual config file at hub.HubDir and hub.ConfigDir +// Handles collections recursively +func (h *Hub) EnableItem(target *Item) error { + var err error + + parentDir := filepath.Clean(h.cfg.InstallDir + "/" + target.Type + "/" + target.Stage + "/") + + // create directories if needed + if target.Installed { + if target.Tainted { + return fmt.Errorf("%s is tainted, won't enable unless --force", target.Name) + } + + if target.Local { + return fmt.Errorf("%s is local, won't enable", target.Name) + } + + // if it's a collection, check sub-items even if the collection file itself is up-to-date + if target.UpToDate && target.Type != COLLECTIONS { + log.Tracef("%s is installed and up-to-date, skip.", target.Name) + return nil + } + } + + if _, err = os.Stat(parentDir); os.IsNotExist(err) { + log.Infof("%s doesn't exist, create", parentDir) + + if err = os.MkdirAll(parentDir, os.ModePerm); err != nil { + return fmt.Errorf("while creating directory: %w", err) + } + } + + // install sub-items if it's a collection + if target.Type == COLLECTIONS { + for _, sub := range target.SubItems() { + val, ok := h.Items[sub.Type][sub.Name] + if !ok { + return fmt.Errorf("required %s %s of %s doesn't exist, abort", sub.Type, sub.Name, target.Name) + } + + err = h.EnableItem(&val) + if err != nil { + return fmt.Errorf("while installing %s: %w", sub.Name, err) + } + } + } + + // check if file already exists where it should in configdir (eg /etc/crowdsec/collections/) + if _, err = os.Lstat(parentDir + "/" + target.FileName); !os.IsNotExist(err) { + log.Infof("%s already exists.", parentDir+"/"+target.FileName) + return nil + } + + // hub.ConfigDir + target.RemotePath + srcPath, err := filepath.Abs(h.cfg.HubDir + "/" + target.RemotePath) + if err != nil { + return fmt.Errorf("while getting source path: %w", err) + } + + dstPath, err := filepath.Abs(parentDir + "/" + target.FileName) + if err != nil { + return fmt.Errorf("while getting destination path: %w", err) + } + + if err = os.Symlink(srcPath, dstPath); err != nil { + return fmt.Errorf("while creating symlink from %s to %s: %w", srcPath, dstPath, err) + } + + log.Infof("Enabled %s : %s", target.Type, target.Name) + target.Installed = true + h.Items[target.Type][target.Name] = *target + + return nil +} + func (h *Hub) purgeItem(target Item) (Item, error) { itempath := h.cfg.HubDir + "/" + target.RemotePath @@ -49,32 +127,31 @@ func (h *Hub) DisableItem(target *Item, purge bool, force bool) error { // for a COLLECTIONS, disable sub-items if target.Type == COLLECTIONS { - for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} { - ptrtype := ItemTypes[idx] - for _, p := range ptr { - if val, ok := h.Items[ptrtype][p]; ok { - // check if the item doesn't belong to another collection before removing it - toRemove := true + for _, sub := range target.SubItems() { + val, ok := h.Items[sub.Type][sub.Name] + if !ok { + log.Errorf("Referred %s %s in collection %s doesn't exist.", sub.Type, sub.Name, target.Name) + continue + } - for _, collection := range val.BelongsToCollections { - if collection != target.Name { - toRemove = false - break - } - } + // check if the item doesn't belong to another collection before removing it + toRemove := true - if toRemove { - err = h.DisableItem(&val, purge, force) - if err != nil { - return fmt.Errorf("while disabling %s: %w", p, err) - } - } else { - log.Infof("%s was not removed because it belongs to another collection", val.Name) - } - } else { - log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, target.Name) + for _, collection := range val.BelongsToCollections { + if collection != target.Name { + toRemove = false + break } } + + if toRemove { + err = h.DisableItem(&val, purge, force) + if err != nil { + return fmt.Errorf("while disabling %s: %w", sub.Name, err) + } + } else { + log.Infof("%s was not removed because it belongs to another collection", val.Name) + } } } @@ -132,81 +209,3 @@ func (h *Hub) DisableItem(target *Item, purge bool, force bool) error { return nil } - -// creates symlink between actual config file at hub.HubDir and hub.ConfigDir -// Handles collections recursively -func (h *Hub) EnableItem(target *Item) error { - var err error - - parentDir := filepath.Clean(h.cfg.InstallDir + "/" + target.Type + "/" + target.Stage + "/") - - // create directories if needed - if target.Installed { - if target.Tainted { - return fmt.Errorf("%s is tainted, won't enable unless --force", target.Name) - } - - if target.Local { - return fmt.Errorf("%s is local, won't enable", target.Name) - } - - // if it's a collection, check sub-items even if the collection file itself is up-to-date - if target.UpToDate && target.Type != COLLECTIONS { - log.Tracef("%s is installed and up-to-date, skip.", target.Name) - return nil - } - } - - if _, err = os.Stat(parentDir); os.IsNotExist(err) { - log.Infof("%s doesn't exist, create", parentDir) - - if err = os.MkdirAll(parentDir, os.ModePerm); err != nil { - return fmt.Errorf("while creating directory: %w", err) - } - } - - // install sub-items if it's a collection - if target.Type == COLLECTIONS { - for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} { - ptrtype := ItemTypes[idx] - for _, p := range ptr { - val, ok := h.Items[ptrtype][p] - if !ok { - return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name) - } - - err = h.EnableItem(&val) - if err != nil { - return fmt.Errorf("while installing %s: %w", p, err) - } - } - } - } - - // check if file already exists where it should in configdir (eg /etc/crowdsec/collections/) - if _, err = os.Lstat(parentDir + "/" + target.FileName); !os.IsNotExist(err) { - log.Infof("%s already exists.", parentDir+"/"+target.FileName) - return nil - } - - // hub.ConfigDir + target.RemotePath - srcPath, err := filepath.Abs(h.cfg.HubDir + "/" + target.RemotePath) - if err != nil { - return fmt.Errorf("while getting source path: %w", err) - } - - dstPath, err := filepath.Abs(parentDir + "/" + target.FileName) - if err != nil { - return fmt.Errorf("while getting destination path: %w", err) - } - - if err = os.Symlink(srcPath, dstPath); err != nil { - return fmt.Errorf("while creating symlink from %s to %s: %w", srcPath, dstPath, err) - } - - log.Infof("Enabled %s : %s", target.Type, target.Name) - target.Installed = true - h.Items[target.Type][target.Name] = *target - - return nil -} diff --git a/pkg/cwhub/enable_test.go b/pkg/cwhub/enable_test.go new file mode 100644 index 000000000..864b3fa85 --- /dev/null +++ b/pkg/cwhub/enable_test.go @@ -0,0 +1,144 @@ +package cwhub + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func testInstall(hub *Hub, t *testing.T, item Item) { + // Install the parser + err := hub.DownloadLatest(&item, false, false) + require.NoError(t, err, "failed to download %s", item.Name) + + _, err = hub.LocalSync() + require.NoError(t, err, "failed to run localSync") + + assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) + + err = hub.EnableItem(&item) + require.NoError(t, err, "failed to enable %s", item.Name) + + _, err = hub.LocalSync() + require.NoError(t, err, "failed to run localSync") + + assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) +} + +func testTaint(hub *Hub, t *testing.T, item Item) { + assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) + + f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600) + require.NoError(t, err, "failed to open %s (%s)", item.LocalPath, item.Name) + + defer f.Close() + + _, err = f.WriteString("tainted") + require.NoError(t, err, "failed to write to %s (%s)", item.LocalPath, item.Name) + + // Local sync and check status + _, err = hub.LocalSync() + require.NoError(t, err, "failed to run localSync") + + assert.True(t, hub.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name) +} + +func testUpdate(hub *Hub, t *testing.T, item Item) { + assert.False(t, hub.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name) + + // Update it + check status + err := hub.DownloadLatest(&item, true, true) + require.NoError(t, err, "failed to update %s", item.Name) + + // Local sync and check status + _, err = hub.LocalSync() + require.NoError(t, err, "failed to run localSync") + + assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) +} + +func testDisable(hub *Hub, t *testing.T, item Item) { + assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) + + // Remove + err := hub.DisableItem(&item, false, false) + require.NoError(t, err, "failed to disable %s", item.Name) + + // Local sync and check status + warns, err := hub.LocalSync() + require.NoError(t, err, "failed to run localSync") + require.Empty(t, warns, "unexpected warnings : %+v", warns) + + assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name) + + // Purge + err = hub.DisableItem(&item, true, false) + require.NoError(t, err, "failed to purge %s", item.Name) + + // Local sync and check status + warns, err = hub.LocalSync() + require.NoError(t, err, "failed to run localSync") + require.Empty(t, warns, "unexpected warnings : %+v", warns) + + assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name) +} + +func TestInstallParser(t *testing.T) { + /* + - install a random parser + - check its status + - taint it + - check its status + - force update it + - check its status + - remove it + */ + hub := envSetup(t) + + // map iteration is random by itself + for _, it := range hub.Items[PARSERS] { + testInstall(hub, t, it) + it = hub.Items[PARSERS][it.Name] + testTaint(hub, t, it) + it = hub.Items[PARSERS][it.Name] + testUpdate(hub, t, it) + it = hub.Items[PARSERS][it.Name] + testDisable(hub, t, it) + + break + } +} + +func TestInstallCollection(t *testing.T) { + /* + - install a random parser + - check its status + - taint it + - check its status + - force update it + - check its status + - remove it + */ + hub := envSetup(t) + + // map iteration is random by itself + for _, it := range hub.Items[COLLECTIONS] { + testInstall(hub, t, it) + it = hub.Items[COLLECTIONS][it.Name] + testTaint(hub, t, it) + it = hub.Items[COLLECTIONS][it.Name] + testUpdate(hub, t, it) + it = hub.Items[COLLECTIONS][it.Name] + testDisable(hub, t, it) + + break + } +} diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index 05c34824d..ad66336be 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -1,65 +1,23 @@ package cwhub +// Install, upgrade and remove items from the hub to the local configuration + +// XXX: this file could use a better name + import ( + "bytes" + "crypto/sha256" "fmt" + "io" + "net/http" + "os" "path/filepath" + "strings" "github.com/enescakir/emoji" log "github.com/sirupsen/logrus" - "golang.org/x/mod/semver" - - "github.com/crowdsecurity/crowdsec/pkg/cwversion" ) -// chooseHubBranch returns the branch name to use for the hub -// It can be "master" or branch corresponding to the current crowdsec version -func chooseHubBranch() string { - latest, err := cwversion.Latest() - if err != nil { - log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err) - return "master" - } - - csVersion := cwversion.VersionStrip() - if csVersion == latest { - log.Debugf("current version is equal to latest (%s)", csVersion) - return "master" - } - - // if current version is greater than the latest we are in pre-release - if semver.Compare(csVersion, latest) == 1 { - log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion) - return "master" - } - - if csVersion == "" { - log.Warning("Crowdsec version is not set, using master branch for the hub") - return "master" - } - - log.Warnf("Crowdsec is not the latest version. "+ - "Current version is '%s' and the latest stable version is '%s'. Please update it!", - csVersion, latest) - - log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+ - "added to Crowdsec Hub after CrowdSec %s", latest) - - return csVersion -} - -// SetHubBranch sets the package variable that points to the hub branch. -func SetHubBranch() { - // a branch is already set, or specified from the flags - if HubBranch != "" { - return - } - - // use the branch corresponding to the crowdsec version - HubBranch = chooseHubBranch() - - log.Debugf("Using branch '%s' for the hub", HubBranch) -} - // InstallItem installs an item from the hub func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly bool) error { item := h.GetItem(itemType, name) @@ -80,7 +38,7 @@ func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly return fmt.Errorf("while downloading %s: %w", item.Name, err) } - if err = h.AddItem(itemType, *item); err != nil { + if err = h.AddItem(*item); err != nil { return fmt.Errorf("while adding %s: %w", item.Name, err) } @@ -94,7 +52,7 @@ func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly return fmt.Errorf("while enabling %s: %w", item.Name, err) } - if err := h.AddItem(itemType, *item); err != nil { + if err := h.AddItem(*item); err != nil { return fmt.Errorf("while adding %s: %w", item.Name, err) } @@ -117,7 +75,7 @@ func (h *Hub) RemoveMany(itemType string, name string, all bool, purge bool, for return fmt.Errorf("unable to disable %s: %w", item.Name, err) } - if err = h.AddItem(itemType, *item); err != nil { + if err = h.AddItem(*item); err != nil { return fmt.Errorf("unable to add %s: %w", item.Name, err) } @@ -141,7 +99,7 @@ func (h *Hub) RemoveMany(itemType string, name string, all bool, purge bool, for return fmt.Errorf("unable to disable %s: %w", v.Name, err) } - if err := h.AddItem(itemType, v); err != nil { + if err := h.AddItem(v); err != nil { return fmt.Errorf("unable to add %s: %w", v.Name, err) } disabled++ @@ -204,7 +162,7 @@ func (h *Hub) UpgradeConfig(itemType string, name string, force bool) error { updated++ } - if err := h.AddItem(itemType, v); err != nil { + if err := h.AddItem(v); err != nil { return fmt.Errorf("unable to add %s: %w", v.Name, err) } } @@ -225,3 +183,192 @@ func (h *Hub) UpgradeConfig(itemType string, name string, force bool) error { return nil } + +// DownloadLatest will download the latest version of Item to the tdir directory +func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) error { + var err error + + log.Debugf("Downloading %s %s", target.Type, target.Name) + + if target.Type != COLLECTIONS { + if !target.Installed && updateOnly && target.Downloaded { + log.Debugf("skipping upgrade of %s : not installed", target.Name) + return nil + } + + return h.DownloadItem(target, overwrite) + } + + // collection + for _, sub := range target.SubItems() { + val, ok := h.Items[sub.Type][sub.Name] + if !ok { + return fmt.Errorf("required %s %s of %s doesn't exist, abort", sub.Type, sub.Name, target.Name) + } + + if !val.Installed && updateOnly && val.Downloaded { + log.Debugf("skipping upgrade of %s : not installed", target.Name) + continue + } + + log.Debugf("Download %s sub-item : %s %s (%t -> %t)", target.Name, sub.Type, sub.Name, target.Installed, updateOnly) + //recurse as it's a collection + if sub.Type == COLLECTIONS { + log.Tracef("collection, recurse") + + err = h.DownloadLatest(&val, overwrite, updateOnly) + if err != nil { + return fmt.Errorf("while downloading %s: %w", val.Name, err) + } + } + + downloaded := val.Downloaded + + err = h.DownloadItem(&val, overwrite) + if err != nil { + return fmt.Errorf("while downloading %s: %w", val.Name, err) + } + + // We need to enable an item when it has been added to a collection since latest release of the collection. + // We check if val.Downloaded is false because maybe the item has been disabled by the user. + if !val.Installed && !downloaded { + if err = h.EnableItem(&val); err != nil { + return fmt.Errorf("enabling '%s': %w", val.Name, err) + } + } + + h.Items[sub.Type][sub.Name] = val + } + + err = h.DownloadItem(target, overwrite) + if err != nil { + return fmt.Errorf("failed to download item: %w", err) + } + + return nil +} + +func (h *Hub) DownloadItem(target *Item, overwrite bool) error { + tdir := h.cfg.HubDir + + // if user didn't --force, don't overwrite local, tainted, up-to-date files + if !overwrite { + if target.Tainted { + log.Debugf("%s : tainted, not updated", target.Name) + return nil + } + + if target.UpToDate { + // We still have to check if data files are present + log.Debugf("%s : up-to-date, not updated", target.Name) + } + } + + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil) + if err != nil { + return fmt.Errorf("while downloading %s: %w", req.URL.String(), err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("while downloading %s: %w", req.URL.String(), err) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String()) + } + + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("while reading %s: %w", req.URL.String(), err) + } + + hash := sha256.New() + if _, err = hash.Write(body); err != nil { + return fmt.Errorf("while hashing %s: %w", target.Name, err) + } + + meow := fmt.Sprintf("%x", hash.Sum(nil)) + if meow != target.Versions[target.Version].Digest { + log.Errorf("Downloaded version doesn't match index, please 'hub update'") + log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest) + + return fmt.Errorf("invalid download hash for %s", target.Name) + } + + //all good, install + //check if parent dir exists + tmpdirs := strings.Split(tdir+"/"+target.RemotePath, "/") + parentDir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/") + + // ensure that target file is within target dir + finalPath, err := filepath.Abs(tdir + "/" + target.RemotePath) + if err != nil { + return fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+target.RemotePath, err) + } + + if !strings.HasPrefix(finalPath, tdir) { + return fmt.Errorf("path %s escapes %s, abort", target.RemotePath, tdir) + } + + // check dir + if _, err = os.Stat(parentDir); os.IsNotExist(err) { + log.Debugf("%s doesn't exist, create", parentDir) + + if err = os.MkdirAll(parentDir, os.ModePerm); err != nil { + return fmt.Errorf("while creating parent directories: %w", err) + } + } + + // check actual file + if _, err = os.Stat(finalPath); !os.IsNotExist(err) { + log.Warningf("%s : overwrite", target.Name) + log.Debugf("target: %s/%s", tdir, target.RemotePath) + } else { + log.Infof("%s : OK", target.Name) + } + + f, err := os.OpenFile(tdir+"/"+target.RemotePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return fmt.Errorf("while opening file: %w", err) + } + + defer f.Close() + + _, err = f.Write(body) + if err != nil { + return fmt.Errorf("while writing file: %w", err) + } + + target.Downloaded = true + target.Tainted = false + target.UpToDate = true + + if err = downloadData(h.cfg.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil { + return fmt.Errorf("while downloading data for %s: %w", target.FileName, err) + } + + h.Items[target.Type][target.Name] = *target + + return nil +} + +// DownloadDataIfNeeded downloads the data files for an item +func (h *Hub) DownloadDataIfNeeded(target Item, force bool) error { + itemFilePath := fmt.Sprintf("%s/%s/%s/%s", h.cfg.InstallDir, target.Type, target.Stage, target.FileName) + + itemFile, err := os.Open(itemFilePath) + if err != nil { + return fmt.Errorf("while opening %s: %w", itemFilePath, err) + } + + defer itemFile.Close() + + if err = downloadData(h.cfg.InstallDataDir, force, itemFile); err != nil { + return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err) + } + + return nil +} diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index 72dac2342..1ece43269 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -14,8 +14,6 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { hub := envSetup(t) // fresh install of collection - hub = getHubOrFail(t, hub.cfg) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) @@ -59,8 +57,6 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) { hub := envSetup(t) // fresh install of collection - hub = getHubOrFail(t, hub.cfg) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) @@ -99,6 +95,7 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) { func getHubOrFail(t *testing.T, hubCfg *csconfig.HubCfg) *Hub { hub, err := InitHub(hubCfg) require.NoError(t, err, "failed to load hub index") + return hub } @@ -109,8 +106,6 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t * hub := envSetup(t) // fresh install of collection - hub = getHubOrFail(t, hub.cfg) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index 6c9dce94b..2a2bfbb0a 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -1,8 +1,13 @@ package cwhub import ( + "bytes" "encoding/json" + "errors" "fmt" + "io" + "net/http" + "os" "strings" log "github.com/sirupsen/logrus" @@ -10,22 +15,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) -const ( - HubIndexFile = ".index.json" - - // managed item types - COLLECTIONS = "collections" - PARSERS = "parsers" - POSTOVERFLOWS = "postoverflows" - SCENARIOS = "scenarios" -) - -var ( - // XXX: The order is important, as it is used to range over sub-items in collections - ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS} -) - -type HubItems map[string]map[string]Item +const HubIndexFile = ".index.json" // Hub represents the runtime status of the hub (parsed items, etc.) type Hub struct { @@ -35,7 +25,10 @@ type Hub struct { skippedTainted int } -var theHub *Hub +var ( + theHub *Hub + ErrIndexNotFound = fmt.Errorf("index not found") +) // GetHub returns the hub singleton // it returns an error if it's not initialized to avoid nil dereference @@ -47,32 +40,129 @@ func GetHub() (*Hub, error) { return theHub, nil } -// displaySummary prints a total count of the hub items -func (h Hub) displaySummary() { - msg := "Loaded: " - for itemType := range h.Items { - msg += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType) +// InitHub initializes the Hub, syncs the local state and returns the singleton for immediate use +func InitHub(cfg *csconfig.HubCfg) (*Hub, error) { + if cfg == nil { + return nil, fmt.Errorf("no configuration found for hub") } - log.Info(strings.Trim(msg, ", ")) - if h.skippedLocal > 0 || h.skippedTainted > 0 { - log.Infof("unmanaged items: %d local, %d tainted", h.skippedLocal, h.skippedTainted) - } -} + log.Debugf("loading hub idx %s", cfg.HubIndexFile) -// DisplaySummary prints a total count of the hub items. -// It is a wrapper around HubIndex.displaySummary() to avoid exporting the hub singleton -// XXX: to be removed later -func DisplaySummary() error { - hub, err := GetHub() + bidx, err := os.ReadFile(cfg.HubIndexFile) if err != nil { - return err + return nil, fmt.Errorf("unable to read index file: %w", err) } - hub.displaySummary() - return nil + + ret, err := ParseIndex(bidx) + if err != nil { + if !errors.Is(err, ErrMissingReference) { + return nil, fmt.Errorf("unable to load existing index: %w", err) + } + + // XXX: why the error check if we bail out anyway? + return nil, err + } + + theHub = &Hub{ + Items: ret, + cfg: cfg, + } + + _, err = theHub.LocalSync() + if err != nil { + return nil, fmt.Errorf("failed to sync Hub index with local deployment : %w", err) + } + + return theHub, nil } -// ParseIndex takes the content of a .index.json file and returns the map of associated parsers/scenarios/collections +// InitHubUpdate is like InitHub but downloads and updates the index instead of reading from the disk +// It is used to inizialize the hub when there is no index file yet +func InitHubUpdate(cfg *csconfig.HubCfg) (*Hub, error) { + if cfg == nil { + return nil, fmt.Errorf("no configuration found for hub") + } + + bidx, err := DownloadIndex(cfg.HubIndexFile) + if err != nil { + return nil, fmt.Errorf("failed to download index: %w", err) + } + + ret, err := ParseIndex(bidx) + if err != nil { + if !errors.Is(err, ErrMissingReference) { + return nil, fmt.Errorf("failed to read index: %w", err) + } + } + + theHub = &Hub{ + Items: ret, + cfg: cfg, + } + + if _, err := theHub.LocalSync(); err != nil { + return nil, fmt.Errorf("failed to sync: %w", err) + } + + return theHub, nil +} + +// DownloadIndex downloads the latest version of the index and returns the content +func DownloadIndex(indexPath string) ([]byte, error) { + log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile)) + + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil) + if err != nil { + return nil, fmt.Errorf("failed to build request for hub index: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed http request for hub index: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusNotFound { + return nil, ErrIndexNotFound + } + + return nil, fmt.Errorf("bad http code %d while requesting %s", resp.StatusCode, req.URL.String()) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read request answer for hub index: %w", err) + } + + oldContent, err := os.ReadFile(indexPath) + if err != nil { + if !os.IsNotExist(err) { + log.Warningf("failed to read hub index: %s", err) + } + } else if bytes.Equal(body, oldContent) { + log.Info("hub index is up to date") + // write it anyway, can't hurt + } + + file, err := os.OpenFile(indexPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + + if err != nil { + return nil, fmt.Errorf("while opening hub index file: %w", err) + } + defer file.Close() + + wsize, err := file.Write(body) + if err != nil { + return nil, fmt.Errorf("while writing hub index file: %w", err) + } + + log.Infof("Wrote new %d bytes index to %s", wsize, indexPath) + + return body, nil +} + +// ParseIndex takes the content of an index file and returns the map of associated parsers/scenarios/collections func ParseIndex(buff []byte) (HubItems, error) { var ( RawIndex HubItems @@ -102,13 +192,10 @@ func ParseIndex(buff []byte) (HubItems, error) { // if it's a collection, check its sub-items are present // XXX should be done later - for idx, ptr := range [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections} { - ptrtype := ItemTypes[idx] - for _, p := range ptr { - if _, ok := RawIndex[ptrtype][p]; !ok { - log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, item.Name) - missingItems = append(missingItems, p) - } + for _, sub := range item.SubItems() { + if _, ok := RawIndex[sub.Type][sub.Name]; !ok { + log.Errorf("Referred %s %s in collection %s doesn't exist.", sub.Type, sub.Name, item.Name) + missingItems = append(missingItems, sub.Name) } } } @@ -120,3 +207,34 @@ func ParseIndex(buff []byte) (HubItems, error) { return RawIndex, nil } + +// ItemStats returns total counts of the hub items +func (h Hub) ItemStats() []string { + loaded := "" + for _, itemType := range ItemTypes { + // ensure the order is always the same + if h.Items[itemType] == nil { + continue + } + if len(h.Items[itemType]) == 0 { + continue + } + loaded += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType) + } + + loaded = strings.Trim(loaded, ", ") + if loaded == "" { + // empty hub + loaded = "0 items" + } + + ret := []string{ + fmt.Sprintf("Loaded: %s", loaded), + } + + if h.skippedLocal > 0 || h.skippedTainted > 0 { + ret = append(ret, fmt.Sprintf("Unmanaged items: %d local, %d tainted", h.skippedLocal, h.skippedTainted)) + } + + return ret +} diff --git a/pkg/cwhub/hub_test.go b/pkg/cwhub/hub_test.go new file mode 100644 index 000000000..1b3a31109 --- /dev/null +++ b/pkg/cwhub/hub_test.go @@ -0,0 +1,63 @@ +package cwhub + +import ( + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/go-cs-lib/cstest" +) + +func TestInitHubUpdate(t *testing.T) { + hub := envSetup(t) + + _, err := InitHubUpdate(hub.cfg) + require.NoError(t, err) + + _, err = GetHub() + require.NoError(t, err) +} + +func TestDownloadIndex(t *testing.T) { + back := RawFileURLTemplate + // bad url template + fmt.Println("Test 'bad URL'") + + tmpIndex, err := os.CreateTemp("", "index.json") + require.NoError(t, err) + + t.Cleanup(func() { + os.Remove(tmpIndex.Name()) + }) + + RawFileURLTemplate = "x" + + ret, err := DownloadIndex(tmpIndex.Name()) + cstest.RequireErrorContains(t, err, "failed to build request for hub index: parse ") + + fmt.Printf("->%+v", ret) + + // bad domain + fmt.Println("Test 'bad domain'") + + RawFileURLTemplate = "https://baddomain/%s/%s" + + ret, err = DownloadIndex(tmpIndex.Name()) + cstest.RequireErrorContains(t, err, "failed http request for hub index: Get") + + fmt.Printf("->%+v", ret) + + // bad target path + fmt.Println("Test 'bad target path'") + + RawFileURLTemplate = back + + ret, err = DownloadIndex("/does/not/exist/index.json") + cstest.RequireErrorContains(t, err, "while opening hub index file: open /does/not/exist/index.json:") + + RawFileURLTemplate = back + + fmt.Printf("->%+v", ret) +} diff --git a/pkg/cwhub/items.go b/pkg/cwhub/items.go new file mode 100644 index 000000000..7d20b107e --- /dev/null +++ b/pkg/cwhub/items.go @@ -0,0 +1,232 @@ +package cwhub + +import ( + "fmt" + + "github.com/enescakir/emoji" + "golang.org/x/mod/semver" +) + +const ( + // managed item types + COLLECTIONS = "collections" + PARSERS = "parsers" + POSTOVERFLOWS = "postoverflows" + SCENARIOS = "scenarios" +) + +// XXX: The order is important, as it is used to range over sub-items in collections +var ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS} + +type HubItems map[string]map[string]Item + +// ItemVersion is used to detect the version of a given item +// by comparing the hash of each version to the local file. +// If the item does not match any known version, it is considered tainted. +type ItemVersion struct { + Digest string `json:"digest,omitempty"` // meow + Deprecated bool `json:"deprecated,omitempty"` // XXX: do we keep this? +} + +// Item represents an object managed in the hub. It can be a parser, scenario, collection.. +type Item struct { + // descriptive info + Type string `json:"type,omitempty" yaml:"type,omitempty"` // can be any of the ItemTypes + Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-... + Name string `json:"name,omitempty"` // as seen in .index.json, usually "author/name" + FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml + Description string `json:"description,omitempty" yaml:"description,omitempty"` // as seen in .index.json + Author string `json:"author,omitempty"` // as seen in .index.json + References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .index.json + BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any + + // remote (hub) info + RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // the path relative to (git | hub API) ie. /parsers/stage/author/file.yaml + Version string `json:"version,omitempty"` // the last version + Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // the list of existing versions + + // local (deployed) info + LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR} + LocalVersion string `json:"local_version,omitempty"` + LocalHash string `json:"local_hash,omitempty"` // the local meow + Installed bool `json:"installed,omitempty"` + Downloaded bool `json:"downloaded,omitempty"` + UpToDate bool `json:"up_to_date,omitempty"` + Tainted bool `json:"tainted,omitempty"` // has it been locally modified? + Local bool `json:"local,omitempty"` // if it's a non versioned control one + + // if it's a collection, it can have sub items + Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"` + PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"` + Scenarios []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"` + Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"` +} + + +type SubItem struct { + Type string + Name string +} + +func (i *Item) SubItems() []SubItem { + sub := make([]SubItem, + len(i.Parsers) + + len(i.PostOverflows) + + len(i.Scenarios) + + len(i.Collections)) + n := 0 + for _, name := range i.Parsers { + sub[n] = SubItem{Type: PARSERS, Name: name} + n++ + } + for _, name := range i.PostOverflows { + sub[n] = SubItem{Type: POSTOVERFLOWS, Name: name} + n++ + } + for _, name := range i.Scenarios { + sub[n] = SubItem{Type: SCENARIOS, Name: name} + n++ + } + for _, name := range i.Collections { + sub[n] = SubItem{Type: COLLECTIONS, Name: name} + n++ + } + return sub +} + +// Status returns the status of the item as a string and an emoji +// ie. "enabled,update-available" and emoji.Warning +func (i *Item) Status() (string, emoji.Emoji) { + status := "disabled" + ok := false + + if i.Installed { + ok = true + status = "enabled" + } + + managed := true + if i.Local { + managed = false + status += ",local" + } + + warning := false + if i.Tainted { + warning = true + status += ",tainted" + } else if !i.UpToDate && !i.Local { + warning = true + status += ",update-available" + } + + emo := emoji.QuestionMark + + switch { + case !managed: + emo = emoji.House + case !i.Installed: + emo = emoji.Prohibited + case warning: + emo = emoji.Warning + case ok: + emo = emoji.CheckMark + } + + return status, emo +} + +// versionStatus: semver requires 'v' prefix +func (i *Item) versionStatus() int { + return semver.Compare("v"+i.Version, "v"+i.LocalVersion) +} + +// validPath returns true if the (relative) path is allowed for the item +// dirNmae: the directory name (ie. crowdsecurity) +// fileName: the filename (ie. apache2-logs.yaml) +func (i *Item) validPath(dirName, fileName string) bool { + return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml") +} + +// GetItemMap returns the map of items for a given type +func (h *Hub) GetItemMap(itemType string) map[string]Item { + m, ok := h.Items[itemType] + if !ok { + return nil + } + + return m +} + +// GetItem returns the item from hub based on its type and full name (author/name) +func (h *Hub) GetItem(itemType string, itemName string) *Item { + m, ok := h.GetItemMap(itemType)[itemName] + if !ok { + return nil + } + + return &m +} + +// GetItemNames returns the list of item (full) names for a given type +// ie. for parsers: crowdsecurity/apache2 crowdsecurity/nginx +// The names can be used to retrieve the item with GetItem() +func (h *Hub) GetItemNames(itemType string) []string { + m := h.GetItemMap(itemType) + if m == nil { + return nil + } + + names := make([]string, 0, len(m)) + for k := range m { + names = append(names, k) + } + + return names +} + +// AddItem adds an item to the hub index +func (h *Hub) AddItem(item Item) error { + for _, t := range ItemTypes { + if t == item.Type { + h.Items[t][item.Name] = item + return nil + } + } + + return fmt.Errorf("ItemType %s is unknown", item.Type) +} + +// GetInstalledItems returns the list of installed items +func (h *Hub) GetInstalledItems(itemType string) ([]Item, error) { + items, ok := h.Items[itemType] + if !ok { + return nil, fmt.Errorf("no %s in hubIdx", itemType) + } + + retItems := make([]Item, 0) + + for _, item := range items { + if item.Installed { + retItems = append(retItems, item) + } + } + + return retItems, nil +} + +// GetInstalledItemsAsString returns the names of the installed items +func (h *Hub) GetInstalledItemsAsString(itemType string) ([]string, error) { + items, err := h.GetInstalledItems(itemType) + if err != nil { + return nil, err + } + + retStr := make([]string, len(items)) + + for i, it := range items { + retStr[i] = it.Name + } + + return retStr, nil +} diff --git a/pkg/cwhub/items_test.go b/pkg/cwhub/items_test.go new file mode 100644 index 000000000..e16dbe312 --- /dev/null +++ b/pkg/cwhub/items_test.go @@ -0,0 +1,75 @@ +package cwhub + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/go-cs-lib/cstest" +) + +func TestItemStatus(t *testing.T) { + hub := envSetup(t) + + // get existing map + x := hub.GetItemMap(COLLECTIONS) + require.NotEmpty(t, x) + + // Get item : good and bad + for k := range x { + item := hub.GetItem(COLLECTIONS, k) + require.NotNil(t, item) + + item.Installed = true + item.UpToDate = false + item.Local = false + item.Tainted = false + + txt, _ := item.Status() + require.Equal(t, "enabled,update-available", txt) + + item.Installed = false + item.UpToDate = false + item.Local = true + item.Tainted = false + + txt, _ = item.Status() + require.Equal(t, "disabled,local", txt) + } + + stats := hub.ItemStats() + require.Equal(t, []string{"Loaded: 2 parsers, 1 scenarios, 3 collections"}, stats) +} + +func TestGetters(t *testing.T) { + hub := envSetup(t) + + // get non existing map + empty := hub.GetItemMap("ratata") + require.Nil(t, empty) + + // get existing map + x := hub.GetItemMap(COLLECTIONS) + require.NotEmpty(t, x) + + // Get item : good and bad + for k := range x { + empty := hub.GetItem(COLLECTIONS, k+"nope") + require.Nil(t, empty) + + item := hub.GetItem(COLLECTIONS, k) + require.NotNil(t, item) + + // Add item and get it + item.Name += "nope" + err := hub.AddItem(*item) + require.NoError(t, err) + + newitem := hub.GetItem(COLLECTIONS, item.Name) + require.NotNil(t, newitem) + + item.Type = "ratata" + err = hub.AddItem(*item) + cstest.RequireErrorContains(t, err, "ItemType ratata is unknown") + } +} diff --git a/pkg/cwhub/leakybucket.go b/pkg/cwhub/leakybucket.go new file mode 100644 index 000000000..fe26aa4c1 --- /dev/null +++ b/pkg/cwhub/leakybucket.go @@ -0,0 +1,58 @@ +package cwhub + +// Resolve a symlink to find the hub item it points to. +// This file is used only by pkg/leakybucket + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary +func itemKey(itemPath string) (string, error) { + f, err := os.Lstat(itemPath) + if err != nil { + return "", fmt.Errorf("while performing lstat on %s: %w", itemPath, err) + } + + if f.Mode()&os.ModeSymlink == 0 { + // it's not a symlink, so the filename itsef should be the key + return filepath.Base(itemPath), nil + } + + // resolve the symlink to hub file + pathInHub, err := os.Readlink(itemPath) + if err != nil { + return "", fmt.Errorf("while reading symlink of %s: %w", itemPath, err) + } + + author := filepath.Base(filepath.Dir(pathInHub)) + + fname := filepath.Base(pathInHub) + fname = strings.TrimSuffix(fname, ".yaml") + fname = strings.TrimSuffix(fname, ".yml") + + return fmt.Sprintf("%s/%s", author, fname), nil +} + +// GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item. +func (h *Hub) GetItemByPath(itemType string, itemPath string) (*Item, error) { + itemKey, err := itemKey(itemPath) + if err != nil { + return nil, err + } + + m := h.GetItemMap(itemType) + if m == nil { + return nil, fmt.Errorf("item type %s doesn't exist", itemType) + } + + v, ok := m[itemKey] + if !ok { + return nil, fmt.Errorf("%s not found in %s", itemKey, itemType) + } + + return &v, nil +} diff --git a/pkg/cwhub/loader.go b/pkg/cwhub/sync.go similarity index 80% rename from pkg/cwhub/loader.go rename to pkg/cwhub/sync.go index 31e347067..a9360a93c 100644 --- a/pkg/cwhub/loader.go +++ b/pkg/cwhub/sync.go @@ -2,7 +2,6 @@ package cwhub import ( "crypto/sha256" - "errors" "fmt" "io" "os" @@ -11,8 +10,6 @@ import ( "strings" log "github.com/sirupsen/logrus" - - "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) func isYAMLFileName(path string) bool { @@ -109,7 +106,7 @@ func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) { log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype) // log.Infof("%s -> name:%s stage:%s", path, fname, stage) - + if ret.stage == SCENARIOS { ret.ftype = SCENARIOS ret.stage = "" @@ -325,66 +322,63 @@ func (h *Hub) CollectDepsCheck(v *Item) error { // if it's a collection, ensure all the items are installed, or tag it as tainted log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed) - for idx, itemSlice := range [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections} { - sliceType := ItemTypes[idx] - for _, subName := range itemSlice { - subItem, ok := h.Items[sliceType][subName] - if !ok { - return fmt.Errorf("referred %s %s in collection %s doesn't exist", sliceType, subName, v.Name) - } - - log.Tracef("check %s installed:%t", subItem.Name, subItem.Installed) - - if !v.Installed { - continue - } - - if subItem.Type == COLLECTIONS { - log.Tracef("collec, recurse.") - - if err := h.CollectDepsCheck(&subItem); err != nil { - if subItem.Tainted { - v.Tainted = true - } - - return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err) - } - - h.Items[sliceType][subName] = subItem - } - - // propagate the state of sub-items to set - if subItem.Tainted { - v.Tainted = true - return fmt.Errorf("tainted %s %s, tainted", sliceType, subName) - } - - if !subItem.Installed && v.Installed { - v.Tainted = true - return fmt.Errorf("missing %s %s, tainted", sliceType, subName) - } - - if !subItem.UpToDate { - v.UpToDate = false - return fmt.Errorf("outdated %s %s", sliceType, subName) - } - - skip := false - - for idx := range subItem.BelongsToCollections { - if subItem.BelongsToCollections[idx] == v.Name { - skip = true - } - } - - if !skip { - subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name) - } - - h.Items[sliceType][subName] = subItem - - log.Tracef("checking for %s - tainted:%t uptodate:%t", subName, v.Tainted, v.UpToDate) + for _, sub := range v.SubItems() { + subItem, ok := h.Items[sub.Type][sub.Name] + if !ok { + return fmt.Errorf("referred %s %s in collection %s doesn't exist", sub.Type, sub.Name, v.Name) } + + log.Tracef("check %s installed:%t", subItem.Name, subItem.Installed) + + if !v.Installed { + continue + } + + if subItem.Type == COLLECTIONS { + log.Tracef("collec, recurse.") + + if err := h.CollectDepsCheck(&subItem); err != nil { + if subItem.Tainted { + v.Tainted = true + } + + return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err) + } + + h.Items[sub.Type][sub.Name] = subItem + } + + // propagate the state of sub-items to set + if subItem.Tainted { + v.Tainted = true + return fmt.Errorf("tainted %s %s, tainted", sub.Type, sub.Name) + } + + if !subItem.Installed && v.Installed { + v.Tainted = true + return fmt.Errorf("missing %s %s, tainted", sub.Type, sub.Name) + } + + if !subItem.UpToDate { + v.UpToDate = false + return fmt.Errorf("outdated %s %s", sub.Type, sub.Name) + } + + skip := false + + for idx := range subItem.BelongsToCollections { + if subItem.BelongsToCollections[idx] == v.Name { + skip = true + } + } + + if !skip { + subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name) + } + + h.Items[sub.Type][sub.Name] = subItem + + log.Tracef("checking for %s - tainted:%t uptodate:%t", sub.Name, v.Tainted, v.UpToDate) } return nil @@ -447,39 +441,3 @@ func (h *Hub) LocalSync() ([]string, error) { return warnings, nil } - -// InitHub initializes the Hub, syncs the local state and returns the singleton for immediate use -func InitHub(cfg *csconfig.HubCfg) (*Hub, error) { - if cfg == nil { - return nil, fmt.Errorf("no configuration found for hub") - } - - log.Debugf("loading hub idx %s", cfg.HubIndexFile) - - bidx, err := os.ReadFile(cfg.HubIndexFile) - if err != nil { - return nil, fmt.Errorf("unable to read index file: %w", err) - } - - ret, err := ParseIndex(bidx) - if err != nil { - if !errors.Is(err, ErrMissingReference) { - return nil, fmt.Errorf("unable to load existing index: %w", err) - } - - // XXX: why the error check if we bail out anyway? - return nil, err - } - - theHub = &Hub{ - Items: ret, - cfg: cfg, - } - - _, err = theHub.LocalSync() - if err != nil { - return nil, fmt.Errorf("failed to sync Hub index with local deployment : %w", err) - } - - return theHub, nil -} diff --git a/pkg/hubtest/utils.go b/pkg/hubtest/utils.go index 5ccbbad39..489b324c4 100644 --- a/pkg/hubtest/utils.go +++ b/pkg/hubtest/utils.go @@ -22,7 +22,7 @@ func Copy(src string, dst string) error { return err } - err = os.WriteFile(dst, content, 0644) + err = os.WriteFile(dst, content, 0o644) if err != nil { return err } diff --git a/test/bats/20_hub_collections.bats b/test/bats/20_hub_collections.bats index db33ae97f..694c92f8a 100644 --- a/test/bats/20_hub_collections.bats +++ b/test/bats/20_hub_collections.bats @@ -258,6 +258,14 @@ teardown() { assert_output "0" } +@test "cscli parsers remove [parser]... --force" { + # remove a parser that belongs to a collection + rune -0 cscli collections install crowdsecurity/linux + rune -0 cscli collections remove crowdsecurity/sshd + assert_stderr --partial "crowdsecurity/sshd belongs to collections: [crowdsecurity/linux]" + assert_stderr --partial "Run 'sudo cscli collections remove crowdsecurity/sshd --force' if you want to force remove this collection" +} + @test "cscli collections upgrade [collection]..." { rune -1 cscli collections upgrade assert_stderr --partial "specify at least one collection to upgrade or '--all'" diff --git a/test/bats/20_hub_collections_dep.bats b/test/bats/20_hub_collections_dep.bats index 26844c965..a703c6596 100644 --- a/test/bats/20_hub_collections_dep.bats +++ b/test/bats/20_hub_collections_dep.bats @@ -49,8 +49,8 @@ teardown() { rune -0 cscli collections install crowdsecurity/smb # XXX: should this be an error? rune -0 cscli collections remove crowdsecurity/sshd - assert_stderr --partial "crowdsecurity/sshd belongs to other collections: [crowdsecurity/smb]" - assert_stderr --partial "Run 'sudo cscli collections remove crowdsecurity/sshd --force' if you want to force remove this sub collection" + assert_stderr --partial "crowdsecurity/sshd belongs to collections: [crowdsecurity/smb]" + assert_stderr --partial "Run 'sudo cscli collections remove crowdsecurity/sshd --force' if you want to force remove this collection" rune -0 cscli collections list -o json rune -0 jq -c '[.collections[].name]' <(output) assert_json '["crowdsecurity/smb","crowdsecurity/sshd"]' diff --git a/test/bats/20_hub_parsers.bats b/test/bats/20_hub_parsers.bats index 77c3dc089..b349cf4e2 100644 --- a/test/bats/20_hub_parsers.bats +++ b/test/bats/20_hub_parsers.bats @@ -268,6 +268,14 @@ teardown() { assert_output "0" } +@test "cscli parsers remove [parser]... --force" { + # remove a parser that belongs to a collection + rune -0 cscli collections install crowdsecurity/linux + rune -0 cscli parsers remove crowdsecurity/sshd-logs + assert_stderr --partial "crowdsecurity/sshd-logs belongs to collections: [crowdsecurity/sshd]" + assert_stderr --partial "Run 'sudo cscli parsers remove crowdsecurity/sshd-logs --force' if you want to force remove this parser" +} + @test "cscli parsers upgrade [parser]..." { rune -1 cscli parsers upgrade assert_stderr --partial "specify at least one parser to upgrade or '--all'" diff --git a/test/bats/20_hub_postoverflows.bats b/test/bats/20_hub_postoverflows.bats index cf74268f7..da0f17b05 100644 --- a/test/bats/20_hub_postoverflows.bats +++ b/test/bats/20_hub_postoverflows.bats @@ -260,6 +260,14 @@ teardown() { assert_output "0" } +@test "cscli postoverflows remove [parser]... --force" { + # remove a parser that belongs to a collection + rune -0 cscli collections install crowdsecurity/auditd + rune -0 cscli postoverflows remove crowdsecurity/auditd-whitelisted-process + assert_stderr --partial "crowdsecurity/auditd-whitelisted-process belongs to collections: [crowdsecurity/auditd]" + assert_stderr --partial "Run 'sudo cscli postoverflows remove crowdsecurity/auditd-whitelisted-process --force' if you want to force remove this postoverflow" +} + @test "cscli postoverflows upgrade [postoverflow]..." { rune -1 cscli postoverflows upgrade assert_stderr --partial "specify at least one postoverflow to upgrade or '--all'" diff --git a/test/bats/20_hub_scenarios.bats b/test/bats/20_hub_scenarios.bats index 2eeb6146b..04c344fe4 100644 --- a/test/bats/20_hub_scenarios.bats +++ b/test/bats/20_hub_scenarios.bats @@ -260,6 +260,14 @@ teardown() { assert_output "0" } +@test "cscli scenarios remove [scenario]... --force" { + # remove a scenario that belongs to a collection + rune -0 cscli collections install crowdsecurity/sshd + rune -0 cscli scenarios remove crowdsecurity/ssh-bf + assert_stderr --partial "crowdsecurity/ssh-bf belongs to collections: [crowdsecurity/sshd]" + assert_stderr --partial "Run 'sudo cscli scenarios remove crowdsecurity/ssh-bf --force' if you want to force remove this scenario" +} + @test "cscli scenarios upgrade [scenario]..." { rune -1 cscli scenarios upgrade assert_stderr --partial "specify at least one scenario to upgrade or '--all'"