diff --git a/.golangci.yml b/.golangci.yml index c6dac451f..e9e426cb6 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -184,7 +184,7 @@ linters: maintidx: # raise this after refactoring - under: 15 + under: 18 misspell: locale: US @@ -210,7 +210,7 @@ linters: - name: cognitive-complexity arguments: # lower this after refactoring - - 119 + - 113 - name: comment-spacings disabled: true - name: confusing-results @@ -235,8 +235,8 @@ linters: - name: function-length arguments: # lower this after refactoring - - 111 - - 238 + - 87 + - 198 - name: get-return disabled: true - name: increment-decrement @@ -294,9 +294,7 @@ linters: - -ST1003 - -ST1005 - -ST1012 - - -ST1022 - -QF1003 - - -QF1008 - -QF1012 wsl: diff --git a/cmd/crowdsec-cli/clialert/alerts.go b/cmd/crowdsec-cli/clialert/alerts.go index 3e69acfec..cd9e636ee 100644 --- a/cmd/crowdsec-cli/clialert/alerts.go +++ b/cmd/crowdsec-cli/clialert/alerts.go @@ -12,6 +12,7 @@ import ( "strconv" "strings" "text/template" + "time" "github.com/fatih/color" "github.com/go-openapi/strfmt" @@ -19,6 +20,7 @@ import ( "github.com/spf13/cobra" "gopkg.in/yaml.v3" + "github.com/crowdsecurity/go-cs-lib/cstime" "github.com/crowdsecurity/go-cs-lib/maptools" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/args" @@ -102,15 +104,15 @@ func (cli *cliAlerts) alertsToTable(alerts *models.GetAlertsResponse, printMachi if *alerts == nil { // avoid returning "null" in json // could be cleaner if we used slice of alerts directly - fmt.Println("[]") + fmt.Fprintln(os.Stdout, "[]") return nil } x, _ := json.MarshalIndent(alerts, "", " ") - fmt.Print(string(x)) + fmt.Fprint(os.Stdout, string(x)) case "human": if len(*alerts) == 0 { - fmt.Println("No active alerts") + fmt.Fprintln(os.Stdout, "No active alerts") return nil } @@ -154,7 +156,7 @@ func (cli *cliAlerts) displayOneAlert(alert *models.Alert, withDetail bool) erro alertDecisionsTable(color.Output, cfg.Cscli.Color, alert) if len(alert.Meta) > 0 { - fmt.Printf("\n - Context :\n") + fmt.Fprintf(os.Stdout, "\n - Context :\n") sort.Slice(alert.Meta, func(i, j int) bool { return alert.Meta[i].Key < alert.Meta[j].Key }) @@ -181,7 +183,7 @@ func (cli *cliAlerts) displayOneAlert(alert *models.Alert, withDetail bool) erro } if withDetail { - fmt.Printf("\n - Events :\n") + fmt.Fprintf(os.Stdout, "\n - Events :\n") for _, event := range alert.Events { alertEventTable(color.Output, cfg.Cscli.Color, event) @@ -238,7 +240,7 @@ func (cli *cliAlerts) NewCommand() *cobra.Command { func (cli *cliAlerts) list(ctx context.Context, alertListFilter apiclient.AlertsListOpts, limit *int, contained *bool, printMachine bool) error { var err error - *alertListFilter.ScopeEquals, err = SanitizeScope(*alertListFilter.ScopeEquals, *alertListFilter.IPEquals, *alertListFilter.RangeEquals) + alertListFilter.ScopeEquals, err = SanitizeScope(alertListFilter.ScopeEquals, alertListFilter.IPEquals, alertListFilter.RangeEquals) if err != nil { return err } @@ -247,66 +249,10 @@ func (cli *cliAlerts) list(ctx context.Context, alertListFilter apiclient.Alerts alertListFilter.Limit = limit } - if *alertListFilter.Until == "" { - alertListFilter.Until = nil - } else if strings.HasSuffix(*alertListFilter.Until, "d") { - /*time.ParseDuration support hours 'h' as bigger unit, let's make the user's life easier*/ - realDuration := strings.TrimSuffix(*alertListFilter.Until, "d") - - days, err := strconv.Atoi(realDuration) - if err != nil { - return fmt.Errorf("can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *alertListFilter.Until) - } - - *alertListFilter.Until = fmt.Sprintf("%d%s", days*24, "h") - } - - if *alertListFilter.Since == "" { - alertListFilter.Since = nil - } else if strings.HasSuffix(*alertListFilter.Since, "d") { - // time.ParseDuration support hours 'h' as bigger unit, let's make the user's life easier - realDuration := strings.TrimSuffix(*alertListFilter.Since, "d") - - days, err := strconv.Atoi(realDuration) - if err != nil { - return fmt.Errorf("can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *alertListFilter.Since) - } - - *alertListFilter.Since = fmt.Sprintf("%d%s", days*24, "h") - } - if *alertListFilter.IncludeCAPI { *alertListFilter.Limit = 0 } - if *alertListFilter.TypeEquals == "" { - alertListFilter.TypeEquals = nil - } - - if *alertListFilter.ScopeEquals == "" { - alertListFilter.ScopeEquals = nil - } - - if *alertListFilter.ValueEquals == "" { - alertListFilter.ValueEquals = nil - } - - if *alertListFilter.ScenarioEquals == "" { - alertListFilter.ScenarioEquals = nil - } - - if *alertListFilter.IPEquals == "" { - alertListFilter.IPEquals = nil - } - - if *alertListFilter.RangeEquals == "" { - alertListFilter.RangeEquals = nil - } - - if *alertListFilter.OriginEquals == "" { - alertListFilter.OriginEquals = nil - } - if contained != nil && *contained { alertListFilter.Contains = new(bool) } @@ -325,16 +271,16 @@ func (cli *cliAlerts) list(ctx context.Context, alertListFilter apiclient.Alerts func (cli *cliAlerts) newListCmd() *cobra.Command { alertListFilter := apiclient.AlertsListOpts{ - ScopeEquals: new(string), - ValueEquals: new(string), - ScenarioEquals: new(string), - IPEquals: new(string), - RangeEquals: new(string), - Since: new(string), - Until: new(string), - TypeEquals: new(string), + ScopeEquals: "", + ValueEquals: "", + ScenarioEquals: "", + IPEquals: "", + RangeEquals: "", + Since: cstime.DurationWithDays(0), + Until: cstime.DurationWithDays(0), + TypeEquals: "", IncludeCAPI: new(bool), - OriginEquals: new(string), + OriginEquals: "", } limit := new(int) @@ -362,15 +308,15 @@ cscli alerts list --type ban`, flags := cmd.Flags() flags.SortFlags = false flags.BoolVarP(alertListFilter.IncludeCAPI, "all", "a", false, "Include decisions from Central API") - flags.StringVar(alertListFilter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)") - flags.StringVar(alertListFilter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)") - flags.StringVarP(alertListFilter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value )") - flags.StringVarP(alertListFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)") - flags.StringVarP(alertListFilter.RangeEquals, "range", "r", "", "restrict to alerts from this range (shorthand for --scope range --value )") - flags.StringVar(alertListFilter.TypeEquals, "type", "", "restrict to alerts with given decision type (ie. ban, captcha)") - flags.StringVar(alertListFilter.ScopeEquals, "scope", "", "restrict to alerts of this scope (ie. ip,range)") - flags.StringVarP(alertListFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope") - flags.StringVar(alertListFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ","))) + flags.Var(&alertListFilter.Until, "until", "restrict to alerts older than until (ie. 4h, 30d)") + flags.Var(&alertListFilter.Since, "since", "restrict to alerts newer than since (ie. 4h, 30d)") + flags.StringVarP(&alertListFilter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value )") + flags.StringVarP(&alertListFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)") + flags.StringVarP(&alertListFilter.RangeEquals, "range", "r", "", "restrict to alerts from this range (shorthand for --scope range --value )") + flags.StringVar(&alertListFilter.TypeEquals, "type", "", "restrict to alerts with given decision type (ie. ban, captcha)") + flags.StringVar(&alertListFilter.ScopeEquals, "scope", "", "restrict to alerts of this scope (ie. ip,range)") + flags.StringVarP(&alertListFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope") + flags.StringVar(&alertListFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ","))) flags.BoolVar(contained, "contained", false, "query decisions contained by range") flags.BoolVarP(&printMachine, "machine", "m", false, "print machines that sent alerts") flags.IntVarP(limit, "limit", "l", 50, "limit size of alerts list table (0 to view all alerts)") @@ -382,7 +328,7 @@ func (cli *cliAlerts) delete(ctx context.Context, delFilter apiclient.AlertsDele var err error if !deleteAll { - *delFilter.ScopeEquals, err = SanitizeScope(*delFilter.ScopeEquals, *delFilter.IPEquals, *delFilter.RangeEquals) + delFilter.ScopeEquals, err = SanitizeScope(delFilter.ScopeEquals, delFilter.IPEquals, delFilter.RangeEquals) if err != nil { return err } @@ -391,26 +337,6 @@ func (cli *cliAlerts) delete(ctx context.Context, delFilter apiclient.AlertsDele delFilter.ActiveDecisionEquals = activeDecision } - if *delFilter.ScopeEquals == "" { - delFilter.ScopeEquals = nil - } - - if *delFilter.ValueEquals == "" { - delFilter.ValueEquals = nil - } - - if *delFilter.ScenarioEquals == "" { - delFilter.ScenarioEquals = nil - } - - if *delFilter.IPEquals == "" { - delFilter.IPEquals = nil - } - - if *delFilter.RangeEquals == "" { - delFilter.RangeEquals = nil - } - if contained != nil && *contained { delFilter.Contains = new(bool) } @@ -448,11 +374,11 @@ func (cli *cliAlerts) newDeleteCmd() *cobra.Command { ) delFilter := apiclient.AlertsDeleteOpts{ - ScopeEquals: new(string), - ValueEquals: new(string), - ScenarioEquals: new(string), - IPEquals: new(string), - RangeEquals: new(string), + ScopeEquals: "", + ValueEquals: "", + ScenarioEquals: "", + IPEquals: "", + RangeEquals: "", } contained := new(bool) @@ -471,9 +397,9 @@ cscli alerts delete -s crowdsecurity/ssh-bf"`, if deleteAll { return nil } - if *delFilter.ScopeEquals == "" && *delFilter.ValueEquals == "" && - *delFilter.ScenarioEquals == "" && *delFilter.IPEquals == "" && - *delFilter.RangeEquals == "" && delAlertByID == "" { + if delFilter.ScopeEquals == "" && delFilter.ValueEquals == "" && + delFilter.ScenarioEquals == "" && delFilter.IPEquals == "" && + delFilter.RangeEquals == "" && delAlertByID == "" { _ = cmd.Usage() return errors.New("at least one filter or --all must be specified") } @@ -487,11 +413,11 @@ cscli alerts delete -s crowdsecurity/ssh-bf"`, flags := cmd.Flags() flags.SortFlags = false - flags.StringVar(delFilter.ScopeEquals, "scope", "", "the scope (ie. ip,range)") - flags.StringVarP(delFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope") - flags.StringVarP(delFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)") - flags.StringVarP(delFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value )") - flags.StringVarP(delFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value )") + flags.StringVar(&delFilter.ScopeEquals, "scope", "", "the scope (ie. ip,range)") + flags.StringVarP(&delFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope") + flags.StringVarP(&delFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)") + flags.StringVarP(&delFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value )") + flags.StringVarP(&delFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value )") flags.StringVar(&delAlertByID, "id", "", "alert ID") flags.BoolVarP(&deleteAll, "all", "a", false, "delete all alerts") flags.BoolVar(contained, "contained", false, "query decisions contained by range") @@ -525,14 +451,14 @@ func (cli *cliAlerts) inspect(ctx context.Context, details bool, alertIDs ...str return fmt.Errorf("unable to serialize alert with id %s: %w", alertID, err) } - fmt.Printf("%s\n", string(data)) + fmt.Fprintln(os.Stdout, string(data)) case "raw": data, err := yaml.Marshal(alert) if err != nil { return fmt.Errorf("unable to serialize alert with id %s: %w", alertID, err) } - fmt.Println(string(data)) + fmt.Fprintln(os.Stdout, string(data)) } } @@ -560,10 +486,9 @@ func (cli *cliAlerts) newInspectCmd() *cobra.Command { } func (cli *cliAlerts) newFlushCmd() *cobra.Command { - var ( - maxItems int - maxAge string - ) + var maxItems int + + maxAge := cstime.DurationWithDays(7 * 24 * time.Hour) cmd := &cobra.Command{ Use: `flush`, @@ -584,7 +509,7 @@ func (cli *cliAlerts) newFlushCmd() *cobra.Command { return err } log.Info("Flushing alerts. !! This may take a long time !!") - err = db.FlushAlerts(ctx, maxAge, maxItems) + err = db.FlushAlerts(ctx, time.Duration(maxAge), maxItems) if err != nil { return fmt.Errorf("unable to flush alerts: %w", err) } @@ -596,7 +521,7 @@ func (cli *cliAlerts) newFlushCmd() *cobra.Command { cmd.Flags().SortFlags = false cmd.Flags().IntVar(&maxItems, "max-items", 5000, "Maximum number of alert items to keep in the database") - cmd.Flags().StringVar(&maxAge, "max-age", "7d", "Maximum age of alert items to keep in the database") + cmd.Flags().Var(&maxAge, "max-age", "Maximum age of alert items to keep in the database") return cmd } diff --git a/cmd/crowdsec-cli/cliallowlists/allowlists.go b/cmd/crowdsec-cli/cliallowlists/allowlists.go index be4b966be..c28ab5c70 100644 --- a/cmd/crowdsec-cli/cliallowlists/allowlists.go +++ b/cmd/crowdsec-cli/cliallowlists/allowlists.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "net/url" + "os" "slices" "strconv" "strings" @@ -283,7 +284,7 @@ func (cli *cliAllowLists) create(ctx context.Context, db *database.Client, name return err } - fmt.Printf("allowlist '%s' created successfully\n", name) + fmt.Fprintf(os.Stdout, "allowlist '%s' created successfully\n", name) return nil } @@ -392,15 +393,15 @@ func (cli *cliAllowLists) delete(ctx context.Context, db *database.Client, name return err } - fmt.Printf("allowlist '%s' deleted successfully\n", name) + fmt.Fprintf(os.Stdout, "allowlist '%s' deleted successfully\n", name) return nil } func (cli *cliAllowLists) newAddCmd() *cobra.Command { var ( - expirationStr string - comment string + expiration cstime.DurationWithDays + comment string ) cmd := &cobra.Command{ @@ -424,25 +425,16 @@ func (cli *cliAllowLists) newAddCmd() *cobra.Command { return err } - var expiration time.Duration - - if expirationStr != "" { - expiration, err = cstime.ParseDuration(expirationStr) - if err != nil { - return err - } - } - name := args[0] values := args[1:] - return cli.add(ctx, db, name, values, expiration, comment) + return cli.add(ctx, db, name, values, time.Duration(expiration), comment) }, } flags := cmd.Flags() - flags.StringVarP(&expirationStr, "expiration", "e", "", "expiration duration") + flags.VarP(&expiration, "expiration", "e", "expiration duration") flags.StringVarP(&comment, "comment", "d", "", "comment for the value") return cmd @@ -484,7 +476,7 @@ func (cli *cliAllowLists) add(ctx context.Context, db *database.Client, name str } if len(toAdd) == 0 { - fmt.Println("no new values for allowlist") + fmt.Fprintln(os.Stdout, "no new values for allowlist") return nil } @@ -494,7 +486,15 @@ func (cli *cliAllowLists) add(ctx context.Context, db *database.Client, name str } if added > 0 { - fmt.Printf("added %d values to allowlist %s\n", added, name) + fmt.Fprintf(os.Stdout, "added %d values to allowlist %s\n", added, name) + } + + deleted, err := db.ApplyAllowlistsToExistingDecisions(ctx) + if err != nil { + return fmt.Errorf("unable to apply allowlists to existing decisions: %w", err) + } + if deleted > 0 { + fmt.Printf("%d decisions deleted by allowlists\n", deleted) } return nil @@ -623,7 +623,7 @@ func (cli *cliAllowLists) remove(ctx context.Context, db *database.Client, name } if len(toRemove) == 0 { - fmt.Println("no value to remove from allowlist") + fmt.Fprintln(os.Stdout, "no value to remove from allowlist") return nil } @@ -633,7 +633,7 @@ func (cli *cliAllowLists) remove(ctx context.Context, db *database.Client, name } if deleted > 0 { - fmt.Printf("removed %d values from allowlist %s", deleted, name) + fmt.Fprintf(os.Stdout, "removed %d values from allowlist %s", deleted, name) } return nil diff --git a/cmd/crowdsec-cli/clibouncer/prune.go b/cmd/crowdsec-cli/clibouncer/prune.go index 3c27efe39..cf70e63f4 100644 --- a/cmd/crowdsec-cli/clibouncer/prune.go +++ b/cmd/crowdsec-cli/clibouncer/prune.go @@ -9,10 +9,14 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" + "github.com/crowdsecurity/go-cs-lib/cstime" + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/args" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/ask" ) +const defaultPruneDuration = 60 * time.Minute + func (cli *cliBouncers) prune(ctx context.Context, duration time.Duration, force bool) error { if duration < 2*time.Minute { if yes, err := ask.YesNo( @@ -59,12 +63,9 @@ func (cli *cliBouncers) prune(ctx context.Context, duration time.Duration, force } func (cli *cliBouncers) newPruneCmd() *cobra.Command { - var ( - duration time.Duration - force bool - ) + var force bool - const defaultDuration = 60 * time.Minute + duration := cstime.DurationWithDays(defaultPruneDuration) cmd := &cobra.Command{ Use: "prune", @@ -74,12 +75,12 @@ func (cli *cliBouncers) newPruneCmd() *cobra.Command { Example: `cscli bouncers prune -d 45m cscli bouncers prune -d 45m --force`, RunE: func(cmd *cobra.Command, _ []string) error { - return cli.prune(cmd.Context(), duration, force) + return cli.prune(cmd.Context(), time.Duration(duration), force) }, } flags := cmd.Flags() - flags.DurationVarP(&duration, "duration", "d", defaultDuration, "duration of time since last pull") + flags.VarP(&duration, "duration", "d", "duration of time since last pull") flags.BoolVar(&force, "force", false, "force prune without asking for confirmation") return cmd diff --git a/cmd/crowdsec-cli/clidecision/decisions.go b/cmd/crowdsec-cli/clidecision/decisions.go index 91f39f421..c5a67582d 100644 --- a/cmd/crowdsec-cli/clidecision/decisions.go +++ b/cmd/crowdsec-cli/clidecision/decisions.go @@ -23,6 +23,8 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" + + "github.com/crowdsecurity/go-cs-lib/cstime" ) type configGetter func() *csconfig.Config @@ -101,22 +103,22 @@ func (cli *cliDecisions) decisionsToTable(alerts *models.GetAlertsResponse, prin if *alerts == nil { // avoid returning "null" in `json" // could be cleaner if we used slice of alerts directly - fmt.Println("[]") + fmt.Fprintln(os.Stdout, "[]") return nil } x, _ := json.MarshalIndent(alerts, "", " ") - fmt.Printf("%s", string(x)) + fmt.Fprintln(os.Stdout, string(x)) case "human": if len(*alerts) == 0 { - fmt.Println("No active decisions") + fmt.Fprintln(os.Stdout, "No active decisions") return nil } cli.decisionsTable(color.Output, alerts, printMachine) if skipped > 0 { - fmt.Printf("%d duplicated entries skipped\n", skipped) + fmt.Fprintf(os.Stdout, "%d duplicated entries skipped\n", skipped) } } @@ -173,7 +175,7 @@ func (cli *cliDecisions) NewCommand() *cobra.Command { func (cli *cliDecisions) list(ctx context.Context, filter apiclient.AlertsListOpts, noSimu *bool, contained *bool, printMachine bool) error { var err error - *filter.ScopeEquals, err = clialert.SanitizeScope(*filter.ScopeEquals, *filter.IPEquals, *filter.RangeEquals) + filter.ScopeEquals, err = clialert.SanitizeScope(filter.ScopeEquals, filter.IPEquals, filter.RangeEquals) if err != nil { return err } @@ -184,67 +186,13 @@ func (cli *cliDecisions) list(ctx context.Context, filter apiclient.AlertsListOp if noSimu != nil && *noSimu { filter.IncludeSimulated = new(bool) } + /* nullify the empty entries to avoid bad filter */ - if *filter.Until == "" { - filter.Until = nil - } else if strings.HasSuffix(*filter.Until, "d") { - /*time.ParseDuration support hours 'h' as bigger unit, let's make the user's life easier*/ - realDuration := strings.TrimSuffix(*filter.Until, "d") - - days, err := strconv.Atoi(realDuration) - if err != nil { - return fmt.Errorf("can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until) - } - - *filter.Until = fmt.Sprintf("%d%s", days*24, "h") - } - - if *filter.Since == "" { - filter.Since = nil - } else if strings.HasSuffix(*filter.Since, "d") { - /*time.ParseDuration support hours 'h' as bigger unit, let's make the user's life easier*/ - realDuration := strings.TrimSuffix(*filter.Since, "d") - - days, err := strconv.Atoi(realDuration) - if err != nil { - return fmt.Errorf("can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Since) - } - - *filter.Since = fmt.Sprintf("%d%s", days*24, "h") - } if *filter.IncludeCAPI { *filter.Limit = 0 } - if *filter.TypeEquals == "" { - filter.TypeEquals = nil - } - - if *filter.ValueEquals == "" { - filter.ValueEquals = nil - } - - if *filter.ScopeEquals == "" { - filter.ScopeEquals = nil - } - - if *filter.ScenarioEquals == "" { - filter.ScenarioEquals = nil - } - - if *filter.IPEquals == "" { - filter.IPEquals = nil - } - - if *filter.RangeEquals == "" { - filter.RangeEquals = nil - } - - if *filter.OriginEquals == "" { - filter.OriginEquals = nil - } - if contained != nil && *contained { filter.Contains = new(bool) } @@ -264,15 +212,15 @@ func (cli *cliDecisions) list(ctx context.Context, filter apiclient.AlertsListOp func (cli *cliDecisions) newListCmd() *cobra.Command { filter := apiclient.AlertsListOpts{ - ValueEquals: new(string), - ScopeEquals: new(string), - ScenarioEquals: new(string), - OriginEquals: new(string), - IPEquals: new(string), - RangeEquals: new(string), - Since: new(string), - Until: new(string), - TypeEquals: new(string), + ValueEquals: "", + ScopeEquals: "", + ScenarioEquals: "", + OriginEquals: "", + IPEquals: "", + RangeEquals: "", + Since: cstime.DurationWithDays(0), + Until: cstime.DurationWithDays(0), + TypeEquals: "", IncludeCAPI: new(bool), Limit: new(int), } @@ -300,15 +248,15 @@ cscli decisions list --origin lists --scenario list_name flags := cmd.Flags() flags.SortFlags = false flags.BoolVarP(filter.IncludeCAPI, "all", "a", false, "Include decisions from Central API") - flags.StringVar(filter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)") - flags.StringVar(filter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)") - flags.StringVarP(filter.TypeEquals, "type", "t", "", "restrict to this decision type (ie. ban,captcha)") - flags.StringVar(filter.ScopeEquals, "scope", "", "restrict to this scope (ie. ip,range,session)") - flags.StringVar(filter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ","))) - flags.StringVarP(filter.ValueEquals, "value", "v", "", "restrict to this value (ie. 1.2.3.4,userName)") - flags.StringVarP(filter.ScenarioEquals, "scenario", "s", "", "restrict to this scenario (ie. crowdsecurity/ssh-bf)") - flags.StringVarP(filter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value )") - flags.StringVarP(filter.RangeEquals, "range", "r", "", "restrict to alerts from this source range (shorthand for --scope range --value )") + flags.Var(&filter.Since, "since", "restrict to alerts newer than since (ie. 4h, 30d)") + flags.Var(&filter.Until, "until", "restrict to alerts older than until (ie. 4h, 30d)") + flags.StringVarP(&filter.TypeEquals, "type", "t", "", "restrict to this decision type (ie. ban,captcha)") + flags.StringVar(&filter.ScopeEquals, "scope", "", "restrict to this scope (ie. ip,range,session)") + flags.StringVar(&filter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ","))) + flags.StringVarP(&filter.ValueEquals, "value", "v", "", "restrict to this value (ie. 1.2.3.4,userName)") + flags.StringVarP(&filter.ScenarioEquals, "scenario", "s", "", "restrict to this scenario (ie. crowdsecurity/ssh-bf)") + flags.StringVarP(&filter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value )") + flags.StringVarP(&filter.RangeEquals, "range", "r", "", "restrict to alerts from this source range (shorthand for --scope range --value )") flags.IntVarP(filter.Limit, "limit", "l", 100, "number of alerts to get (use 0 to remove the limit)") flags.BoolVar(NoSimu, "no-simu", false, "exclude decisions in simulation mode") flags.BoolVarP(&printMachine, "machine", "m", false, "print machines that triggered decisions") @@ -452,39 +400,11 @@ func (cli *cliDecisions) delete(ctx context.Context, delFilter apiclient.Decisio var err error /*take care of shorthand options*/ - *delFilter.ScopeEquals, err = clialert.SanitizeScope(*delFilter.ScopeEquals, *delFilter.IPEquals, *delFilter.RangeEquals) + delFilter.ScopeEquals, err = clialert.SanitizeScope(delFilter.ScopeEquals, delFilter.IPEquals, delFilter.RangeEquals) if err != nil { return err } - if *delFilter.ScopeEquals == "" { - delFilter.ScopeEquals = nil - } - - if *delFilter.OriginEquals == "" { - delFilter.OriginEquals = nil - } - - if *delFilter.ValueEquals == "" { - delFilter.ValueEquals = nil - } - - if *delFilter.ScenarioEquals == "" { - delFilter.ScenarioEquals = nil - } - - if *delFilter.TypeEquals == "" { - delFilter.TypeEquals = nil - } - - if *delFilter.IPEquals == "" { - delFilter.IPEquals = nil - } - - if *delFilter.RangeEquals == "" { - delFilter.RangeEquals = nil - } - if contained != nil && *contained { delFilter.Contains = new(bool) } @@ -514,13 +434,13 @@ func (cli *cliDecisions) delete(ctx context.Context, delFilter apiclient.Decisio func (cli *cliDecisions) newDeleteCmd() *cobra.Command { delFilter := apiclient.DecisionsDeleteOpts{ - ScopeEquals: new(string), - ValueEquals: new(string), - TypeEquals: new(string), - IPEquals: new(string), - RangeEquals: new(string), - ScenarioEquals: new(string), - OriginEquals: new(string), + ScopeEquals: "", + ValueEquals: "", + TypeEquals: "", + IPEquals: "", + RangeEquals: "", + ScenarioEquals: "", + OriginEquals: "", } var delDecisionID string @@ -546,10 +466,10 @@ cscli decisions delete --origin lists --scenario list_name if delDecisionAll { return nil } - if *delFilter.ScopeEquals == "" && *delFilter.ValueEquals == "" && - *delFilter.TypeEquals == "" && *delFilter.IPEquals == "" && - *delFilter.RangeEquals == "" && *delFilter.ScenarioEquals == "" && - *delFilter.OriginEquals == "" && delDecisionID == "" { + if delFilter.ScopeEquals == "" && delFilter.ValueEquals == "" && + delFilter.TypeEquals == "" && delFilter.IPEquals == "" && + delFilter.RangeEquals == "" && delFilter.ScenarioEquals == "" && + delFilter.OriginEquals == "" && delDecisionID == "" { _ = cmd.Usage() return errors.New("at least one filter or --all must be specified") } @@ -563,12 +483,12 @@ cscli decisions delete --origin lists --scenario list_name flags := cmd.Flags() flags.SortFlags = false - flags.StringVarP(delFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value )") - flags.StringVarP(delFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value )") - flags.StringVarP(delFilter.TypeEquals, "type", "t", "", "the decision type (ie. ban,captcha)") - flags.StringVarP(delFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope") - flags.StringVarP(delFilter.ScenarioEquals, "scenario", "s", "", "the scenario name (ie. crowdsecurity/ssh-bf)") - flags.StringVar(delFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ","))) + flags.StringVarP(&delFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value )") + flags.StringVarP(&delFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value )") + flags.StringVarP(&delFilter.TypeEquals, "type", "t", "", "the decision type (ie. ban,captcha)") + flags.StringVarP(&delFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope") + flags.StringVarP(&delFilter.ScenarioEquals, "scenario", "s", "", "the scenario name (ie. crowdsecurity/ssh-bf)") + flags.StringVar(&delFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ","))) flags.StringVar(&delDecisionID, "id", "", "decision id") flags.BoolVar(&delDecisionAll, "all", false, "delete all decisions") diff --git a/cmd/crowdsec-cli/climachine/prune.go b/cmd/crowdsec-cli/climachine/prune.go index 4054305f4..be9102428 100644 --- a/cmd/crowdsec-cli/climachine/prune.go +++ b/cmd/crowdsec-cli/climachine/prune.go @@ -9,11 +9,15 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" + "github.com/crowdsecurity/go-cs-lib/cstime" + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/args" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/ask" "github.com/crowdsecurity/crowdsec/pkg/database/ent" ) +const defaultPruneDuration = 10 * time.Minute + func (cli *cliMachines) prune(ctx context.Context, duration time.Duration, notValidOnly bool, force bool) error { if duration < 2*time.Minute && !notValidOnly { if yes, err := ask.YesNo( @@ -67,12 +71,11 @@ func (cli *cliMachines) prune(ctx context.Context, duration time.Duration, notVa func (cli *cliMachines) newPruneCmd() *cobra.Command { var ( - duration time.Duration notValidOnly bool force bool ) - const defaultDuration = 10 * time.Minute + duration := cstime.DurationWithDays(defaultPruneDuration) cmd := &cobra.Command{ Use: "prune", @@ -84,12 +87,12 @@ cscli machines prune --not-validated-only --force`, Args: args.NoArgs, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, _ []string) error { - return cli.prune(cmd.Context(), duration, notValidOnly, force) + return cli.prune(cmd.Context(), time.Duration(duration), notValidOnly, force) }, } flags := cmd.Flags() - flags.DurationVarP(&duration, "duration", "d", defaultDuration, "duration of time since validated machine last heartbeat") + flags.VarP(&duration, "duration", "d", "duration of time since validated machine last heartbeat") flags.BoolVar(¬ValidOnly, "not-validated-only", false, "only prune machines that are not validated") flags.BoolVar(&force, "force", false, "force prune without asking for confirmation") diff --git a/go.mod b/go.mod index 2d0822753..0d4badd52 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/creack/pty v1.1.21 // indirect github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 - github.com/crowdsecurity/go-cs-lib v0.0.18 + github.com/crowdsecurity/go-cs-lib v0.0.19 github.com/crowdsecurity/go-onnxruntime v0.0.0-20240801073851-3fd7de0127b4 github.com/crowdsecurity/grokky v0.2.2 github.com/crowdsecurity/machineid v1.0.2 @@ -89,8 +89,8 @@ require ( github.com/shirou/gopsutil/v3 v3.23.5 github.com/sirupsen/logrus v1.9.3 github.com/slack-go/slack v0.16.0 - github.com/spf13/cobra v1.8.1 - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/cobra v1.9.1 + github.com/spf13/pflag v1.0.6 // indirect github.com/stretchr/testify v1.10.0 github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 github.com/wasilibs/go-re2 v1.7.0 @@ -133,7 +133,7 @@ require ( github.com/bytedance/sonic/loader v0.2.1 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gabriel-vasile/mimetype v1.4.7 // indirect github.com/gin-contrib/sse v0.1.0 // indirect diff --git a/go.sum b/go.sum index ae79aba7c..84efdd899 100644 --- a/go.sum +++ b/go.sum @@ -100,8 +100,8 @@ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7 github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= @@ -111,8 +111,8 @@ github.com/crowdsecurity/coraza/v3 v3.0.0-20250320231801-749b8bded21a h1:2Nyr+47 github.com/crowdsecurity/coraza/v3 v3.0.0-20250320231801-749b8bded21a/go.mod h1:xSaXWOhFMSbrV8qOOfBKAyw3aOqfwaSaOy5BgSF8XlA= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= -github.com/crowdsecurity/go-cs-lib v0.0.18 h1:GNyvaag5MXfuapIy4E30pIOvIE5AyHoanJBNSMA1cmE= -github.com/crowdsecurity/go-cs-lib v0.0.18/go.mod h1:XwGcvTt4lMq4Tm1IRMSKMDf0CVrnytTU8Uoofa7AR+g= +github.com/crowdsecurity/go-cs-lib v0.0.19 h1:wA4O8hGrEntTGn7eZTJqnQ3mrAje5JvQAj8DNbe5IZg= +github.com/crowdsecurity/go-cs-lib v0.0.19/go.mod h1:hz2FOHFXc0vWzH78uxo2VebtPQ9Snkbdzy3TMA20tVQ= github.com/crowdsecurity/go-onnxruntime v0.0.0-20240801073851-3fd7de0127b4 h1:CwzISIxoKp0dJLrJJIlhvQPuzirpS9QH07guxK5LIeg= github.com/crowdsecurity/go-onnxruntime v0.0.0-20240801073851-3fd7de0127b4/go.mod h1:YfyL16lx2wA8Z6t/TG1x1/FBngOIpuCuo7nM/FSuP54= github.com/crowdsecurity/grokky v0.2.2 h1:yALsI9zqpDArYzmSSxfBq2dhYuGUTKMJq8KOEIAsuo4= @@ -664,11 +664,12 @@ github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= diff --git a/pkg/apiclient/alerts_service.go b/pkg/apiclient/alerts_service.go index 1f84862a8..4d30e5976 100644 --- a/pkg/apiclient/alerts_service.go +++ b/pkg/apiclient/alerts_service.go @@ -7,42 +7,44 @@ import ( qs "github.com/google/go-querystring/query" + "github.com/crowdsecurity/go-cs-lib/cstime" + "github.com/crowdsecurity/crowdsec/pkg/models" ) type AlertsService service type AlertsListOpts struct { - ScopeEquals *string `url:"scope,omitempty"` - ValueEquals *string `url:"value,omitempty"` - ScenarioEquals *string `url:"scenario,omitempty"` - IPEquals *string `url:"ip,omitempty"` - RangeEquals *string `url:"range,omitempty"` - OriginEquals *string `url:"origin,omitempty"` - Since *string `url:"since,omitempty"` - TypeEquals *string `url:"decision_type,omitempty"` - Until *string `url:"until,omitempty"` - IncludeSimulated *bool `url:"simulated,omitempty"` - ActiveDecisionEquals *bool `url:"has_active_decision,omitempty"` - IncludeCAPI *bool `url:"include_capi,omitempty"` - Limit *int `url:"limit,omitempty"` - Contains *bool `url:"contains,omitempty"` + ScopeEquals string `url:"scope,omitempty"` + ValueEquals string `url:"value,omitempty"` + ScenarioEquals string `url:"scenario,omitempty"` + IPEquals string `url:"ip,omitempty"` + RangeEquals string `url:"range,omitempty"` + OriginEquals string `url:"origin,omitempty"` + Since cstime.DurationWithDays `url:"since,omitempty"` + TypeEquals string `url:"decision_type,omitempty"` + Until cstime.DurationWithDays `url:"until,omitempty"` + IncludeSimulated *bool `url:"simulated,omitempty"` + ActiveDecisionEquals *bool `url:"has_active_decision,omitempty"` + IncludeCAPI *bool `url:"include_capi,omitempty"` + Limit *int `url:"limit,omitempty"` + Contains *bool `url:"contains,omitempty"` ListOpts } type AlertsDeleteOpts struct { - ScopeEquals *string `url:"scope,omitempty"` - ValueEquals *string `url:"value,omitempty"` - ScenarioEquals *string `url:"scenario,omitempty"` - IPEquals *string `url:"ip,omitempty"` - RangeEquals *string `url:"range,omitempty"` - Since *string `url:"since,omitempty"` - Until *string `url:"until,omitempty"` - OriginEquals *string `url:"origin,omitempty"` - ActiveDecisionEquals *bool `url:"has_active_decision,omitempty"` - SourceEquals *string `url:"alert_source,omitempty"` - Contains *bool `url:"contains,omitempty"` - Limit *int `url:"limit,omitempty"` + ScopeEquals string `url:"scope,omitempty"` + ValueEquals string `url:"value,omitempty"` + ScenarioEquals string `url:"scenario,omitempty"` + IPEquals string `url:"ip,omitempty"` + RangeEquals string `url:"range,omitempty"` + Since cstime.DurationWithDays `url:"since,omitempty"` + Until cstime.DurationWithDays `url:"until,omitempty"` + OriginEquals string `url:"origin,omitempty"` + ActiveDecisionEquals *bool `url:"has_active_decision,omitempty"` + SourceEquals string `url:"alert_source,omitempty"` + Contains *bool `url:"contains,omitempty"` + Limit *int `url:"limit,omitempty"` ListOpts } diff --git a/pkg/apiclient/alerts_service_test.go b/pkg/apiclient/alerts_service_test.go index 24b66937f..eed1598c8 100644 --- a/pkg/apiclient/alerts_service_test.go +++ b/pkg/apiclient/alerts_service_test.go @@ -18,6 +18,7 @@ import ( func TestAlertsListAsMachine(t *testing.T) { ctx := t.Context() + log.SetLevel(log.DebugLevel) mux, urlx, teardown := setup() @@ -189,7 +190,7 @@ func TestAlertsListAsMachine(t *testing.T) { assert.Equal(t, expected, *alerts) // this one doesn't - filter := AlertsListOpts{IPEquals: ptr.Of("1.2.3.4")} + filter := AlertsListOpts{IPEquals: "1.2.3.4"} alerts, resp, err = client.Alerts.List(ctx, filter) require.NoError(t, err) @@ -199,6 +200,7 @@ func TestAlertsListAsMachine(t *testing.T) { func TestAlertsGetAsMachine(t *testing.T) { ctx := t.Context() + log.SetLevel(log.DebugLevel) mux, urlx, teardown := setup() @@ -367,6 +369,7 @@ func TestAlertsGetAsMachine(t *testing.T) { func TestAlertsCreateAsMachine(t *testing.T) { ctx := t.Context() + log.SetLevel(log.DebugLevel) mux, urlx, teardown := setup() @@ -410,6 +413,7 @@ func TestAlertsCreateAsMachine(t *testing.T) { func TestAlertsDeleteAsMachine(t *testing.T) { ctx := t.Context() + log.SetLevel(log.DebugLevel) mux, urlx, teardown := setup() @@ -442,7 +446,7 @@ func TestAlertsDeleteAsMachine(t *testing.T) { defer teardown() - alert := AlertsDeleteOpts{IPEquals: ptr.Of("1.2.3.4")} + alert := AlertsDeleteOpts{IPEquals: "1.2.3.4"} alerts, resp, err := client.Alerts.Delete(ctx, alert) require.NoError(t, err) diff --git a/pkg/apiclient/auth_key_test.go b/pkg/apiclient/auth_key_test.go index aa92e03bb..1e5f95acc 100644 --- a/pkg/apiclient/auth_key_test.go +++ b/pkg/apiclient/auth_key_test.go @@ -10,11 +10,11 @@ import ( "github.com/stretchr/testify/require" "github.com/crowdsecurity/go-cs-lib/cstest" - "github.com/crowdsecurity/go-cs-lib/ptr" ) func TestApiAuth(t *testing.T) { ctx := t.Context() + log.SetLevel(log.TraceLevel) mux, urlx, teardown := setup() @@ -40,7 +40,7 @@ func TestApiAuth(t *testing.T) { defer teardown() - //ok no answer + // ok no answer auth := &APIKeyTransport{ APIKey: "ixu", } @@ -48,12 +48,12 @@ func TestApiAuth(t *testing.T) { newcli, err := NewDefaultClient(apiURL, "v1", "toto", auth.Client()) require.NoError(t, err) - alert := DecisionsListOpts{IPEquals: ptr.Of("1.2.3.4")} + alert := DecisionsListOpts{IPEquals: "1.2.3.4"} _, resp, err := newcli.Decisions.List(ctx, alert) require.NoError(t, err) assert.Equal(t, http.StatusOK, resp.Response.StatusCode) - //ko bad token + // ko bad token auth = &APIKeyTransport{ APIKey: "bad", } @@ -69,7 +69,7 @@ func TestApiAuth(t *testing.T) { cstest.RequireErrorMessage(t, err, "API error: access forbidden") - //ko empty token + // ko empty token auth = &APIKeyTransport{} newcli, err = NewDefaultClient(apiURL, "v1", "toto", auth.Client()) diff --git a/pkg/apiclient/decisions_service.go b/pkg/apiclient/decisions_service.go index a1810e831..c222e2ddb 100644 --- a/pkg/apiclient/decisions_service.go +++ b/pkg/apiclient/decisions_service.go @@ -20,12 +20,12 @@ import ( type DecisionsService service type DecisionsListOpts struct { - ScopeEquals *string `url:"scope,omitempty"` - ValueEquals *string `url:"value,omitempty"` - TypeEquals *string `url:"type,omitempty"` - IPEquals *string `url:"ip,omitempty"` - RangeEquals *string `url:"range,omitempty"` - Contains *bool `url:"contains,omitempty"` + ScopeEquals string `url:"scope,omitempty"` + ValueEquals string `url:"value,omitempty"` + TypeEquals string `url:"type,omitempty"` + IPEquals string `url:"ip,omitempty"` + RangeEquals string `url:"range,omitempty"` + Contains *bool `url:"contains,omitempty"` ListOpts } @@ -60,15 +60,15 @@ func (o *DecisionsStreamOpts) addQueryParamsToURL(url string) (string, error) { } type DecisionsDeleteOpts struct { - ScopeEquals *string `url:"scope,omitempty"` - ValueEquals *string `url:"value,omitempty"` - TypeEquals *string `url:"type,omitempty"` - IPEquals *string `url:"ip,omitempty"` - RangeEquals *string `url:"range,omitempty"` - Contains *bool `url:"contains,omitempty"` - OriginEquals *string `url:"origin,omitempty"` + ScopeEquals string `url:"scope,omitempty"` + ValueEquals string `url:"value,omitempty"` + TypeEquals string `url:"type,omitempty"` + IPEquals string `url:"ip,omitempty"` + RangeEquals string `url:"range,omitempty"` + Contains *bool `url:"contains,omitempty"` + OriginEquals string `url:"origin,omitempty"` // - ScenarioEquals *string `url:"scenario,omitempty"` + ScenarioEquals string `url:"scenario,omitempty"` ListOpts } diff --git a/pkg/apiclient/decisions_service_test.go b/pkg/apiclient/decisions_service_test.go index c16abed64..c9e555e92 100644 --- a/pkg/apiclient/decisions_service_test.go +++ b/pkg/apiclient/decisions_service_test.go @@ -19,6 +19,7 @@ import ( func TestDecisionsList(t *testing.T) { ctx := t.Context() + log.SetLevel(log.DebugLevel) mux, urlx, teardown := setup() @@ -64,15 +65,13 @@ func TestDecisionsList(t *testing.T) { } // OK decisions - decisionsFilter := DecisionsListOpts{IPEquals: ptr.Of("1.2.3.4")} - decisions, resp, err := newcli.Decisions.List(ctx, decisionsFilter) + decisions, resp, err := newcli.Decisions.List(ctx, DecisionsListOpts{IPEquals: "1.2.3.4"}) require.NoError(t, err) assert.Equal(t, http.StatusOK, resp.Response.StatusCode) assert.Equal(t, *expected, *decisions) // Empty return - decisionsFilter = DecisionsListOpts{IPEquals: ptr.Of("1.2.3.5")} - decisions, resp, err = newcli.Decisions.List(ctx, decisionsFilter) + decisions, resp, err = newcli.Decisions.List(ctx, DecisionsListOpts{IPEquals: "1.2.3.5"}) require.NoError(t, err) assert.Equal(t, http.StatusOK, resp.Response.StatusCode) assert.Empty(t, *decisions) @@ -80,6 +79,7 @@ func TestDecisionsList(t *testing.T) { func TestDecisionsStream(t *testing.T) { ctx := t.Context() + log.SetLevel(log.DebugLevel) mux, urlx, teardown := setup() @@ -156,6 +156,7 @@ func TestDecisionsStream(t *testing.T) { func TestDecisionsStreamV3Compatibility(t *testing.T) { ctx := t.Context() + log.SetLevel(log.DebugLevel) mux, urlx, teardown := setupWithPrefix("v3") @@ -224,6 +225,7 @@ func TestDecisionsStreamV3Compatibility(t *testing.T) { func TestDecisionsStreamV3(t *testing.T) { ctx := t.Context() + log.SetLevel(log.DebugLevel) mux, urlx, teardown := setupWithPrefix("v3") @@ -297,6 +299,7 @@ func TestDecisionsStreamV3(t *testing.T) { func TestDecisionsFromBlocklist(t *testing.T) { ctx := t.Context() + log.SetLevel(log.DebugLevel) mux, urlx, teardown := setupWithPrefix("v3") @@ -429,10 +432,7 @@ func TestDeleteDecisions(t *testing.T) { }) require.NoError(t, err) - filters := DecisionsDeleteOpts{IPEquals: new(string)} - *filters.IPEquals = "1.2.3.4" - - deleted, _, err := client.Decisions.Delete(ctx, filters) + deleted, _, err := client.Decisions.Delete(ctx, DecisionsDeleteOpts{IPEquals: "1.2.3.4"}) require.NoError(t, err) assert.Equal(t, "1", deleted.NbDeleted) } diff --git a/pkg/apiserver/apic.go b/pkg/apiserver/apic.go index 0de773ab4..8e92dc674 100644 --- a/pkg/apiserver/apic.go +++ b/pkg/apiserver/apic.go @@ -11,13 +11,13 @@ import ( "net/http" "net/url" "slices" - "strconv" "strings" "sync" "time" "github.com/davecgh/go-spew/spew" "github.com/go-openapi/strfmt" + "github.com/golang-jwt/jwt/v4" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" @@ -214,8 +214,6 @@ func NewAPIC(ctx context.Context, config *csconfig.OnlineApiClientCfg, dbClient shareSignals: *config.Sharing, } - password := strfmt.Password(config.Credentials.Password) - apiURL, err := url.Parse(config.Credentials.URL) if err != nil { return nil, fmt.Errorf("while parsing '%s': %w", config.Credentials.URL, err) @@ -233,7 +231,7 @@ func NewAPIC(ctx context.Context, config *csconfig.OnlineApiClientCfg, dbClient ret.apiClient, err = apiclient.NewClient(&apiclient.Config{ MachineID: config.Credentials.Login, - Password: password, + Password: strfmt.Password(config.Credentials.Password), URL: apiURL, PapiURL: papiURL, VersionPrefix: "v3", @@ -244,29 +242,103 @@ func NewAPIC(ctx context.Context, config *csconfig.OnlineApiClientCfg, dbClient return nil, fmt.Errorf("while creating api client: %w", err) } - // The watcher will be authenticated by the RoundTripper the first time it will call CAPI - // Explicit authentication will provoke a useless supplementary call to CAPI - scenarios, err := ret.FetchScenariosListFromDB(ctx) + err = ret.Authenticate(ctx, config) + return ret, err +} + +// loadAPICToken attempts to retrieve and validate a JWT token from the local database. +// It returns the token string, its expiration time, and a boolean indicating whether the token is valid. +// +// A token is considered valid if: +// - it exists in the database, +// - it is a properly formatted JWT with an "exp" claim, +// - it is not expired or near expiry. +func loadAPICToken(ctx context.Context, db *database.Client) (string, time.Time, bool) { + token, err := db.GetConfigItem(ctx, "apic_token") if err != nil { - return ret, fmt.Errorf("get scenario in db: %w", err) + log.Debugf("error fetching token from DB: %s", err) + return "", time.Time{}, false } - authResp, _, err := ret.apiClient.Auth.AuthenticateWatcher(ctx, models.WatcherAuthRequest{ + if token == nil { + log.Debug("no token found in DB") + return "", time.Time{}, false + } + + parser := new(jwt.Parser) + tok, _, err := parser.ParseUnverified(*token, jwt.MapClaims{}) + if err != nil { + log.Debugf("error parsing token: %s", err) + return "", time.Time{}, false + } + + claims, ok := tok.Claims.(jwt.MapClaims) + if !ok { + log.Debugf("error parsing token claims: %s", err) + return "", time.Time{}, false + } + + expFloat, ok := claims["exp"].(float64) + if !ok { + log.Debug("token missing 'exp' claim") + return "", time.Time{}, false + } + + exp := time.Unix(int64(expFloat), 0) + if time.Now().UTC().After(exp.Add(-1*time.Minute)) { + log.Debug("auth token expired") + return "", time.Time{}, false + } + + return *token, exp, true +} + +// saveAPICToken stores the given JWT token in the local database under the "apic_token" config item. +func saveAPICToken(ctx context.Context, db *database.Client, token string) error { + if err := db.SetConfigItem(ctx, "apic_token", token); err != nil { + return fmt.Errorf("saving token to db: %w", err) + } + + return nil +} + +// Authenticate ensures the API client is authorized to communicate with the CAPI. +// It attempts to reuse a previously saved JWT token from the database, falling back to +// an authentication request if the token is missing, invalid, or expired. +// +// If a new token is obtained, it is saved back to the database for caching. +func (a *apic) Authenticate(ctx context.Context, config *csconfig.OnlineApiClientCfg) error { + if token, exp, valid := loadAPICToken(ctx, a.dbClient); valid { + log.Debug("using valid token from DB") + a.apiClient.GetClient().Transport.(*apiclient.JWTTransport).Token = token + a.apiClient.GetClient().Transport.(*apiclient.JWTTransport).Expiration = exp + } + + log.Debug("No token found, authenticating") + + scenarios, err := a.FetchScenariosListFromDB(ctx) + if err != nil { + return fmt.Errorf("get scenario in db: %w", err) + } + + password := strfmt.Password(config.Credentials.Password) + + authResp, _, err := a.apiClient.Auth.AuthenticateWatcher(ctx, models.WatcherAuthRequest{ MachineID: &config.Credentials.Login, Password: &password, Scenarios: scenarios, }) if err != nil { - return ret, fmt.Errorf("authenticate watcher (%s): %w", config.Credentials.Login, err) + return fmt.Errorf("authenticate watcher (%s): %w", config.Credentials.Login, err) } - if err = ret.apiClient.GetClient().Transport.(*apiclient.JWTTransport).Expiration.UnmarshalText([]byte(authResp.Expire)); err != nil { - return ret, fmt.Errorf("unable to parse jwt expiration: %w", err) + if err = a.apiClient.GetClient().Transport.(*apiclient.JWTTransport).Expiration.UnmarshalText([]byte(authResp.Expire)); err != nil { + return fmt.Errorf("unable to parse jwt expiration: %w", err) } - ret.apiClient.GetClient().Transport.(*apiclient.JWTTransport).Token = authResp.Token + a.apiClient.GetClient().Transport.(*apiclient.JWTTransport).Token = authResp.Token - return ret, err + return saveAPICToken(ctx, a.dbClient, authResp.Token) } // keep track of all alerts in cache and push it to CAPI every PushInterval. @@ -439,16 +511,11 @@ func (a *apic) HandleDeletedDecisionsV3(ctx context.Context, deletedDecisions [] filter["scopes"] = []string{*scope} } - dbCliRet, _, err := a.dbClient.ExpireDecisionsWithFilter(ctx, filter) + dbCliDel, _, err := a.dbClient.ExpireDecisionsWithFilter(ctx, filter) if err != nil { return 0, fmt.Errorf("expiring decisions error: %w", err) } - dbCliDel, err := strconv.Atoi(dbCliRet) - if err != nil { - return 0, fmt.Errorf("converting db ret %d: %w", dbCliDel, err) - } - updateCounterForDecision(deleteCounters, ptr.Of(types.CAPIOrigin), nil, dbCliDel) nbDeleted += dbCliDel } @@ -588,6 +655,8 @@ func fillAlertsWithDecisions(alerts []*models.Alert, decisions []*models.Decisio func (a *apic) PullTop(ctx context.Context, forcePull bool) error { var err error + hasPulledAllowlists := false + // A mutex with TryLock would be a bit simpler // But go does not guarantee that TryLock will be able to acquire the lock even if it is available select { @@ -649,7 +718,7 @@ func (a *apic) PullTop(ctx context.Context, forcePull bool) error { // process deleted decisions nbDeleted, err := a.HandleDeletedDecisionsV3(ctx, data.Deleted, deleteCounters) if err != nil { - return err + log.Errorf("could not delete decisions from CAPI: %s", err) } log.Printf("capi/community-blocklist : %d explicit deletions", nbDeleted) @@ -657,8 +726,10 @@ func (a *apic) PullTop(ctx context.Context, forcePull bool) error { // Update allowlists before processing decisions if data.Links != nil { if len(data.Links.Allowlists) > 0 { + hasPulledAllowlists = true + if err := a.UpdateAllowlists(ctx, data.Links.Allowlists, forcePull); err != nil { - return fmt.Errorf("while updating allowlists: %w", err) + log.Errorf("could not update allowlists from CAPI: %s", err) } } } @@ -675,7 +746,7 @@ func (a *apic) PullTop(ctx context.Context, forcePull bool) error { err = a.SaveAlerts(ctx, alertsFromCapi, addCounters, deleteCounters) if err != nil { - return fmt.Errorf("while saving alerts: %w", err) + log.Errorf("could not save alert for CAPI pull: %s", err) } } else { if a.pullCommunity { @@ -689,11 +760,22 @@ func (a *apic) PullTop(ctx context.Context, forcePull bool) error { if data.Links != nil { if len(data.Links.Blocklists) > 0 { if err := a.UpdateBlocklists(ctx, data.Links.Blocklists, addCounters, forcePull); err != nil { - return fmt.Errorf("while updating blocklists: %w", err) + log.Errorf("could not update blocklists from CAPI: %s", err) } } } + if hasPulledAllowlists { + deleted, err := a.dbClient.ApplyAllowlistsToExistingDecisions(ctx) + if err != nil { + log.Errorf("could not apply allowlists to existing decisions: %s", err) + } + + if deleted > 0 { + log.Infof("deleted %d decisions from allowlists", deleted) + } + } + return nil } diff --git a/pkg/apiserver/apiserver_test.go b/pkg/apiserver/apiserver_test.go index 01a8588df..6cb0e3d54 100644 --- a/pkg/apiserver/apiserver_test.go +++ b/pkg/apiserver/apiserver_test.go @@ -18,6 +18,7 @@ import ( "github.com/stretchr/testify/require" "github.com/crowdsecurity/go-cs-lib/cstest" + "github.com/crowdsecurity/go-cs-lib/cstime" "github.com/crowdsecurity/go-cs-lib/ptr" "github.com/crowdsecurity/go-cs-lib/version" @@ -47,9 +48,9 @@ var ( func LoadTestConfig(t *testing.T) csconfig.Config { config := csconfig.Config{} - maxAge := "1h" + maxAge := cstime.DurationWithDays(1*time.Hour) flushConfig := csconfig.FlushDBCfg{ - MaxAge: &maxAge, + MaxAge: maxAge, } tempDir, _ := os.MkdirTemp("", "crowdsec_tests") @@ -97,9 +98,9 @@ func LoadTestConfig(t *testing.T) csconfig.Config { func LoadTestConfigForwardedFor(t *testing.T) csconfig.Config { config := csconfig.Config{} - maxAge := "1h" + maxAge := cstime.DurationWithDays(1*time.Hour) flushConfig := csconfig.FlushDBCfg{ - MaxAge: &maxAge, + MaxAge: maxAge, } tempDir, _ := os.MkdirTemp("", "crowdsec_tests") @@ -363,9 +364,9 @@ func TestLoggingDebugToFileConfig(t *testing.T) { ctx := t.Context() /*declare settings*/ - maxAge := "1h" + maxAge := cstime.DurationWithDays(1*time.Hour) flushConfig := csconfig.FlushDBCfg{ - MaxAge: &maxAge, + MaxAge: maxAge, } tempDir, _ := os.MkdirTemp("", "crowdsec_tests") @@ -416,9 +417,9 @@ func TestLoggingErrorToFileConfig(t *testing.T) { ctx := t.Context() /*declare settings*/ - maxAge := "1h" + maxAge := cstime.DurationWithDays(1*time.Hour) flushConfig := csconfig.FlushDBCfg{ - MaxAge: &maxAge, + MaxAge: maxAge, } tempDir, _ := os.MkdirTemp("", "crowdsec_tests") diff --git a/pkg/apiserver/controllers/v1/decisions.go b/pkg/apiserver/controllers/v1/decisions.go index 6a316d8a2..86dd98450 100644 --- a/pkg/apiserver/controllers/v1/decisions.go +++ b/pkg/apiserver/controllers/v1/decisions.go @@ -134,7 +134,7 @@ func (c *Controller) DeleteDecisions(gctx *gin.Context) { } deleteDecisionResp := models.DeleteDecisionResponse{ - NbDeleted: nbDeleted, + NbDeleted: strconv.Itoa(nbDeleted), } gctx.JSON(http.StatusOK, deleteDecisionResp) diff --git a/pkg/apiserver/papi_cmd.go b/pkg/apiserver/papi_cmd.go index 2e48ef4ea..f9ecbf359 100644 --- a/pkg/apiserver/papi_cmd.go +++ b/pkg/apiserver/papi_cmd.go @@ -264,6 +264,14 @@ func ManagementCmd(message *Message, p *Papi, sync bool) error { if err != nil { return fmt.Errorf("failed to force pull operation: %w", err) } + + deleted, err := p.DBClient.ApplyAllowlistsToExistingDecisions(ctx) + if err != nil { + log.Errorf("could not apply allowlists to existing decisions: %s", err) + } + if deleted > 0 { + log.Infof("deleted %d decisions from allowlists", deleted) + } } case "allowlist_unsubscribe": data, err := json.Marshal(message.Data) diff --git a/pkg/csconfig/database.go b/pkg/csconfig/database.go index 26150eb2e..f34b40889 100644 --- a/pkg/csconfig/database.go +++ b/pkg/csconfig/database.go @@ -14,6 +14,7 @@ import ( "github.com/go-sql-driver/mysql" log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/go-cs-lib/cstime" "github.com/crowdsecurity/go-cs-lib/ptr" "github.com/crowdsecurity/crowdsec/pkg/types" @@ -58,10 +59,10 @@ type AuthGCCfg struct { type FlushDBCfg struct { MaxItems *int `yaml:"max_items,omitempty"` // We could unmarshal as time.Duration, but alert filters right now are a map of strings - MaxAge *string `yaml:"max_age,omitempty"` - BouncersGC *AuthGCCfg `yaml:"bouncers_autodelete,omitempty"` - AgentsGC *AuthGCCfg `yaml:"agents_autodelete,omitempty"` - MetricsMaxAge *time.Duration `yaml:"metrics_max_age,omitempty"` + MaxAge cstime.DurationWithDays `yaml:"max_age,omitempty"` + BouncersGC *AuthGCCfg `yaml:"bouncers_autodelete,omitempty"` + AgentsGC *AuthGCCfg `yaml:"agents_autodelete,omitempty"` + MetricsMaxAge cstime.DurationWithDays `yaml:"metrics_max_age,omitempty"` } func (c *Config) LoadDBConfig(inCli bool) error { diff --git a/pkg/csprofiles/csprofiles.go b/pkg/csprofiles/csprofiles.go index 0b2ae0280..3081f4d31 100644 --- a/pkg/csprofiles/csprofiles.go +++ b/pkg/csprofiles/csprofiles.go @@ -7,12 +7,13 @@ import ( "github.com/expr-lang/expr/vm" log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/go-cs-lib/cstime" + "github.com/crowdsecurity/go-cs-lib/ptr" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" - utils "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/crowdsecurity/go-cs-lib/ptr" ) type Runtime struct { @@ -84,7 +85,7 @@ func NewProfile(profilesCfg []*csconfig.ProfileCfg) ([]*Runtime, error) { duration = defaultDuration } - if _, err := utils.ParseDuration(duration); err != nil { + if _, err := cstime.ParseDurationWithDays(duration); err != nil { return nil, fmt.Errorf("error parsing duration '%s' of %s: %w", duration, profile.Name, err) } } @@ -136,7 +137,7 @@ func (profile *Runtime) GenerateDecisionFromProfile(alert *models.Alert) ([]*mod profile.Logger.Warningf("Failed to run duration_expr : %v", err) } else { durationStr := fmt.Sprint(duration) - if _, err := utils.ParseDuration(durationStr); err != nil { + if _, err := cstime.ParseDurationWithDays(durationStr); err != nil { profile.Logger.Warningf("Failed to parse expr duration result '%s'", duration) } else { *decision.Duration = durationStr diff --git a/pkg/database/alertfilter.go b/pkg/database/alertfilter.go index 6ff2ab99a..d96690140 100644 --- a/pkg/database/alertfilter.go +++ b/pkg/database/alertfilter.go @@ -9,6 +9,8 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/go-cs-lib/cstime" + "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/alert" "github.com/crowdsecurity/crowdsec/pkg/database/ent/decision" @@ -40,7 +42,9 @@ func handleScopeFilter(scope string, predicates *[]predicate.Alert) { } func handleTimeFilters(param, value string, predicates *[]predicate.Alert) error { - duration, err := ParseDuration(value) + // crowsdec now always sends duration without days, but we allow them for + // compatibility with other tools + duration, err := cstime.ParseDurationWithDays(value) if err != nil { return fmt.Errorf("while parsing duration: %w", err) } diff --git a/pkg/database/alerts.go b/pkg/database/alerts.go index 6f27b5afe..2fcbb8a5f 100644 --- a/pkg/database/alerts.go +++ b/pkg/database/alerts.go @@ -13,6 +13,7 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/go-cs-lib/cstime" "github.com/crowdsecurity/go-cs-lib/slicetools" "github.com/crowdsecurity/crowdsec/pkg/database/ent" @@ -382,7 +383,7 @@ func (c *Client) createDecisionChunk(ctx context.Context, simulated bool, stopAt sz int ) - duration, err := ParseDuration(*decisionItem.Duration) + duration, err := cstime.ParseDurationWithDays(*decisionItem.Duration) if err != nil { return nil, errors.Wrapf(ParseDurationFail, "decision duration '%+v' : %s", *decisionItem.Duration, err) } diff --git a/pkg/database/allowlists.go b/pkg/database/allowlists.go index d14958f61..a8815a7f6 100644 --- a/pkg/database/allowlists.go +++ b/pkg/database/allowlists.go @@ -12,6 +12,8 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/allowlist" "github.com/crowdsecurity/crowdsec/pkg/database/ent/allowlistitem" + "github.com/crowdsecurity/crowdsec/pkg/database/ent/decision" + "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -389,3 +391,96 @@ func (c *Client) GetAllowlistsContentForAPIC(ctx context.Context) ([]net.IP, []* return ips, nets, nil } + +func (c *Client) ApplyAllowlistsToExistingDecisions(ctx context.Context) (int, error) { + // Soft delete (set expiration to now) all decisions that matches any allowlist + + totalCount := 0 + + // Get all non-expired allowlist items + // We will match them one by one against all decisions + allowlistItems, err := c.Ent.AllowListItem.Query(). + Where( + allowlistitem.Or( + allowlistitem.ExpiresAtGTE(time.Now().UTC()), + allowlistitem.ExpiresAtIsNil(), + ), + ).All(ctx) + if err != nil { + return 0, fmt.Errorf("unable to get allowlist items: %w", err) + } + + now := time.Now().UTC() + + for _, item := range allowlistItems { + updateQuery := c.Ent.Decision.Update().SetUntil(now).Where(decision.UntilGTE(now)) + switch item.IPSize { + case 4: + updateQuery = updateQuery.Where( + decision.And( + decision.IPSizeEQ(4), + decision.Or( + decision.And( + decision.StartIPLTE(item.StartIP), + decision.EndIPGTE(item.EndIP), + ), + decision.And( + decision.StartIPGTE(item.StartIP), + decision.EndIPLTE(item.EndIP), + ), + ))) + case 16: + updateQuery = updateQuery.Where( + decision.And( + decision.IPSizeEQ(16), + decision.Or( + decision.And( + decision.Or( + decision.StartIPLT(item.StartIP), + decision.And( + decision.StartIPEQ(item.StartIP), + decision.StartSuffixLTE(item.StartSuffix), + )), + decision.Or( + decision.EndIPGT(item.EndIP), + decision.And( + decision.EndIPEQ(item.EndIP), + decision.EndSuffixGTE(item.EndSuffix), + ), + ), + ), + decision.And( + decision.Or( + decision.StartIPGT(item.StartIP), + decision.And( + decision.StartIPEQ(item.StartIP), + decision.StartSuffixGTE(item.StartSuffix), + )), + decision.Or( + decision.EndIPLT(item.EndIP), + decision.And( + decision.EndIPEQ(item.EndIP), + decision.EndSuffixLTE(item.EndSuffix), + ), + ), + ), + ), + ), + ) + default: + // This should never happen + // But better safe than sorry and just skip it instead of expiring all decisions + c.Log.Errorf("unexpected IP size %d for allowlist item %s", item.IPSize, item.Value) + continue + } + // Update the decisions + count, err := updateQuery.Save(ctx) + if err != nil { + c.Log.Errorf("unable to expire existing decisions: %s", err) + continue + } + totalCount += count + } + + return totalCount, nil +} diff --git a/pkg/database/decisions.go b/pkg/database/decisions.go index 94b8a54b7..52d0e341c 100644 --- a/pkg/database/decisions.go +++ b/pkg/database/decisions.go @@ -29,9 +29,9 @@ type DecisionsByScenario struct { func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string][]string) (*ent.DecisionQuery, error) { var ( - err error + err error start_ip, start_sfx, end_ip, end_sfx int64 - ip_sz int + ip_sz int ) contains := true @@ -100,18 +100,21 @@ func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string] if err != nil { return nil, errors.Wrapf(InvalidFilter, "invalid limit value : %s", err) } + query = query.Limit(limit) case "offset": offset, err := strconv.Atoi(value[0]) if err != nil { return nil, errors.Wrapf(InvalidFilter, "invalid offset value : %s", err) } + query = query.Offset(offset) case "id_gt": id, err := strconv.Atoi(value[0]) if err != nil { return nil, errors.Wrapf(InvalidFilter, "invalid id_gt value : %s", err) } + query = query.Where(decision.IDGT(id)) } } @@ -201,7 +204,7 @@ func (c *Client) QueryDecisionCountByScenario(ctx context.Context) ([]*Decisions func (c *Client) QueryDecisionWithFilter(ctx context.Context, filter map[string][]string) ([]*ent.Decision, error) { var ( - err error + err error data []*ent.Decision ) @@ -322,70 +325,12 @@ func (c *Client) QueryNewDecisionsSinceWithFilters(ctx context.Context, since *t return data, nil } -func (c *Client) DeleteDecisionsWithFilter(ctx context.Context, filter map[string][]string) (string, []*ent.Decision, error) { - var ( - err error - start_ip, start_sfx, end_ip, end_sfx int64 - ip_sz int - ) - - contains := true - /*if contains is true, return bans that *contains* the given value (value is the inner) - else, return bans that are *contained* by the given value (value is the outer) */ - - decisions := c.Ent.Decision.Query() - - for param, value := range filter { - switch param { - case "contains": - contains, err = strconv.ParseBool(value[0]) - if err != nil { - return "0", nil, errors.Wrapf(InvalidFilter, "invalid contains value : %s", err) - } - case "scope": - decisions = decisions.Where(decision.ScopeEQ(value[0])) - case "value": - decisions = decisions.Where(decision.ValueEQ(value[0])) - case "type": - decisions = decisions.Where(decision.TypeEQ(value[0])) - case "ip", "range": - ip_sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(value[0]) - if err != nil { - return "0", nil, errors.Wrapf(InvalidIPOrRange, "unable to convert '%s' to int: %s", value[0], err) - } - case "scenario": - decisions = decisions.Where(decision.ScenarioEQ(value[0])) - default: - return "0", nil, errors.Wrap(InvalidFilter, fmt.Sprintf("'%s' doesn't exist", param)) - } - } - - decisions, err = decisionIPFilter(decisions, contains, ip_sz, start_ip, start_sfx, end_ip, end_sfx) - if err != nil { - return "0", nil, err - } - - toDelete, err := decisions.All(ctx) - if err != nil { - c.Log.Warningf("DeleteDecisionsWithFilter : %s", err) - return "0", nil, errors.Wrap(DeleteFail, "decisions with provided filter") - } - - count, err := c.DeleteDecisions(ctx, toDelete) - if err != nil { - c.Log.Warningf("While deleting decisions : %s", err) - return "0", nil, errors.Wrap(DeleteFail, "decisions with provided filter") - } - - return strconv.Itoa(count), toDelete, nil -} - // ExpireDecisionsWithFilter updates the expiration time to now() for the decisions matching the filter, and returns the updated items -func (c *Client) ExpireDecisionsWithFilter(ctx context.Context, filter map[string][]string) (string, []*ent.Decision, error) { +func (c *Client) ExpireDecisionsWithFilter(ctx context.Context, filter map[string][]string) (int, []*ent.Decision, error) { var ( - err error + err error start_ip, start_sfx, end_ip, end_sfx int64 - ip_sz int + ip_sz int ) contains := true @@ -398,7 +343,7 @@ func (c *Client) ExpireDecisionsWithFilter(ctx context.Context, filter map[strin case "contains": contains, err = strconv.ParseBool(value[0]) if err != nil { - return "0", nil, errors.Wrapf(InvalidFilter, "invalid contains value : %s", err) + return 0, nil, errors.Wrapf(InvalidFilter, "invalid contains value : %s", err) } case "scopes": decisions = decisions.Where(decision.ScopeEQ(value[0])) @@ -413,32 +358,32 @@ func (c *Client) ExpireDecisionsWithFilter(ctx context.Context, filter map[strin case "ip", "range": ip_sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(value[0]) if err != nil { - return "0", nil, errors.Wrapf(InvalidIPOrRange, "unable to convert '%s' to int: %s", value[0], err) + return 0, nil, errors.Wrapf(InvalidIPOrRange, "unable to convert '%s' to int: %s", value[0], err) } case "scenario": decisions = decisions.Where(decision.ScenarioEQ(value[0])) default: - return "0", nil, errors.Wrapf(InvalidFilter, "'%s' doesn't exist", param) + return 0, nil, errors.Wrapf(InvalidFilter, "'%s' doesn't exist", param) } } decisions, err = decisionIPFilter(decisions, contains, ip_sz, start_ip, start_sfx, end_ip, end_sfx) if err != nil { - return "0", nil, err + return 0, nil, err } DecisionsToDelete, err := decisions.All(ctx) if err != nil { c.Log.Warningf("ExpireDecisionsWithFilter : %s", err) - return "0", nil, errors.Wrap(DeleteFail, "expire decisions with provided filter") + return 0, nil, errors.Wrap(DeleteFail, "expire decisions with provided filter") } count, err := c.ExpireDecisions(ctx, DecisionsToDelete) if err != nil { - return "0", nil, errors.Wrapf(DeleteFail, "expire decisions with provided filter : %s", err) + return 0, nil, errors.Wrapf(DeleteFail, "expire decisions with provided filter : %s", err) } - return strconv.Itoa(count), DecisionsToDelete, err + return count, DecisionsToDelete, err } func decisionIDs(decisions []*ent.Decision) []int { @@ -564,13 +509,7 @@ func (c *Client) CountDecisionsByValue(ctx context.Context, value string, since } func (c *Client) GetActiveDecisionsTimeLeftByValue(ctx context.Context, decisionValue string) (time.Duration, error) { - var ( - err error - start_ip, start_sfx, end_ip, end_sfx int64 - ip_sz int - ) - - ip_sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(decisionValue) + ip_sz, start_ip, start_sfx, end_ip, end_sfx, err := types.Addr2Ints(decisionValue) if err != nil { return 0, fmt.Errorf("unable to convert '%s' to int: %w", decisionValue, err) } diff --git a/pkg/database/ent/migrate/schema.go b/pkg/database/ent/migrate/schema.go index 932c27dd7..571c04af8 100644 --- a/pkg/database/ent/migrate/schema.go +++ b/pkg/database/ent/migrate/schema.go @@ -148,7 +148,7 @@ var ( {Name: "created_at", Type: field.TypeTime}, {Name: "updated_at", Type: field.TypeTime}, {Name: "name", Type: field.TypeString, Unique: true}, - {Name: "value", Type: field.TypeString}, + {Name: "value", Type: field.TypeString, SchemaType: map[string]string{"mysql": "longtext", "postgres": "text"}}, } // ConfigItemsTable holds the schema information for the "config_items" table. ConfigItemsTable = &schema.Table{ diff --git a/pkg/database/ent/schema/config.go b/pkg/database/ent/schema/config.go index d526db25a..2f12f4491 100644 --- a/pkg/database/ent/schema/config.go +++ b/pkg/database/ent/schema/config.go @@ -2,6 +2,7 @@ package schema import ( "entgo.io/ent" + "entgo.io/ent/dialect" "entgo.io/ent/schema/field" "github.com/crowdsecurity/crowdsec/pkg/types" @@ -22,7 +23,10 @@ func (ConfigItem) Fields() []ent.Field { Default(types.UtcNow). UpdateDefault(types.UtcNow).StructTag(`json:"updated_at"`), field.String("name").Unique().StructTag(`json:"name"`).Immutable(), - field.String("value").StructTag(`json:"value"`), // a json object + field.String("value").SchemaType(map[string]string{ + dialect.MySQL: "longtext", + dialect.Postgres: "text", + }).StructTag(`json:"value"`), // a json object } } diff --git a/pkg/database/flush.go b/pkg/database/flush.go index e1b5f9f44..e508c53c0 100644 --- a/pkg/database/flush.go +++ b/pkg/database/flush.go @@ -9,7 +9,7 @@ import ( "github.com/go-co-op/gocron" log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/go-cs-lib/ptr" + "github.com/crowdsecurity/go-cs-lib/cstime" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/database/ent/alert" @@ -30,7 +30,6 @@ const ( func (c *Client) StartFlushScheduler(ctx context.Context, config *csconfig.FlushDBCfg) (*gocron.Scheduler, error) { maxItems := 0 - maxAge := "" if config.MaxItems != nil && *config.MaxItems <= 0 { return nil, errors.New("max_items can't be zero or negative") @@ -40,14 +39,10 @@ func (c *Client) StartFlushScheduler(ctx context.Context, config *csconfig.Flush maxItems = *config.MaxItems } - if config.MaxAge != nil && *config.MaxAge != "" { - maxAge = *config.MaxAge - } - // Init & Start cronjob every minute for alerts scheduler := gocron.NewScheduler(time.UTC) - job, err := scheduler.Every(1).Minute().Do(c.FlushAlerts, ctx, maxAge, maxItems) + job, err := scheduler.Every(1).Minute().Do(c.FlushAlerts, ctx, time.Duration(config.MaxAge), maxItems) if err != nil { return nil, fmt.Errorf("while starting FlushAlerts scheduler: %w", err) } @@ -56,7 +51,7 @@ func (c *Client) StartFlushScheduler(ctx context.Context, config *csconfig.Flush // Init & Start cronjob every hour for bouncers/agents if config.AgentsGC != nil { if config.AgentsGC.Cert != nil { - duration, err := ParseDuration(*config.AgentsGC.Cert) + duration, err := cstime.ParseDurationWithDays(*config.AgentsGC.Cert) if err != nil { return nil, fmt.Errorf("while parsing agents cert auto-delete duration: %w", err) } @@ -65,7 +60,7 @@ func (c *Client) StartFlushScheduler(ctx context.Context, config *csconfig.Flush } if config.AgentsGC.LoginPassword != nil { - duration, err := ParseDuration(*config.AgentsGC.LoginPassword) + duration, err := cstime.ParseDurationWithDays(*config.AgentsGC.LoginPassword) if err != nil { return nil, fmt.Errorf("while parsing agents login/password auto-delete duration: %w", err) } @@ -80,7 +75,7 @@ func (c *Client) StartFlushScheduler(ctx context.Context, config *csconfig.Flush if config.BouncersGC != nil { if config.BouncersGC.Cert != nil { - duration, err := ParseDuration(*config.BouncersGC.Cert) + duration, err := cstime.ParseDurationWithDays(*config.BouncersGC.Cert) if err != nil { return nil, fmt.Errorf("while parsing bouncers cert auto-delete duration: %w", err) } @@ -89,7 +84,7 @@ func (c *Client) StartFlushScheduler(ctx context.Context, config *csconfig.Flush } if config.BouncersGC.Api != nil { - duration, err := ParseDuration(*config.BouncersGC.Api) + duration, err := cstime.ParseDurationWithDays(*config.BouncersGC.Api) if err != nil { return nil, fmt.Errorf("while parsing bouncers api auto-delete duration: %w", err) } @@ -109,7 +104,7 @@ func (c *Client) StartFlushScheduler(ctx context.Context, config *csconfig.Flush baJob.SingletonMode() - metricsJob, err := scheduler.Every(flushInterval).Do(c.flushMetrics, ctx, config.MetricsMaxAge) + metricsJob, err := scheduler.Every(flushInterval).Do(c.flushMetrics, ctx, time.Duration(config.MetricsMaxAge)) if err != nil { return nil, fmt.Errorf("while starting flushMetrics scheduler: %w", err) } @@ -129,15 +124,15 @@ func (c *Client) StartFlushScheduler(ctx context.Context, config *csconfig.Flush } // flushMetrics deletes metrics older than maxAge, regardless if they have been pushed to CAPI or not -func (c *Client) flushMetrics(ctx context.Context, maxAge *time.Duration) { - if maxAge == nil { - maxAge = ptr.Of(defaultMetricsMaxAge) +func (c *Client) flushMetrics(ctx context.Context, maxAge time.Duration) { + if maxAge == 0 { + maxAge = defaultMetricsMaxAge } c.Log.Debugf("flushing metrics older than %s", maxAge) deleted, err := c.Ent.Metric.Delete().Where( - metric.ReceivedAtLTE(time.Now().UTC().Add(-*maxAge)), + metric.ReceivedAtLTE(time.Now().UTC().Add(-maxAge)), ).Exec(ctx) if err != nil { c.Log.Errorf("while flushing metrics: %s", err) @@ -230,7 +225,7 @@ func (c *Client) FlushAgentsAndBouncers(ctx context.Context, agentsCfg *csconfig return nil } -func (c *Client) FlushAlerts(ctx context.Context, maxAge string, maxItems int) error { +func (c *Client) FlushAlerts(ctx context.Context, maxAge time.Duration, maxItems int) error { var ( deletedByAge int deletedByNbItem int @@ -255,9 +250,9 @@ func (c *Client) FlushAlerts(ctx context.Context, maxAge string, maxItems int) e c.Log.Debugf("FlushAlerts (Total alerts): %d", totalAlerts) - if maxAge != "" { + if maxAge != 0 { filter := map[string][]string{ - "created_before": {maxAge}, + "created_before": {maxAge.String()}, } nbDeleted, err := c.DeleteAlertWithFilter(ctx, filter) diff --git a/pkg/database/utils.go b/pkg/database/utils.go index 8148df56f..9b8d20dcf 100644 --- a/pkg/database/utils.go +++ b/pkg/database/utils.go @@ -4,9 +4,6 @@ import ( "encoding/binary" "fmt" "net" - "strconv" - "strings" - "time" ) func IP2Int(ip net.IP) uint32 { @@ -69,28 +66,3 @@ func GetIpsFromIpRange(host string) (int64, int64, error) { return ipStart, ipEnd, nil } - -func ParseDuration(d string) (time.Duration, error) { - durationStr := d - - if strings.HasSuffix(d, "d") { - days := strings.Split(d, "d")[0] - if days == "" { - return 0, fmt.Errorf("'%s' can't be parsed as duration", d) - } - - daysInt, err := strconv.Atoi(days) - if err != nil { - return 0, err - } - - durationStr = strconv.Itoa(daysInt*24) + "h" - } - - duration, err := time.ParseDuration(durationStr) - if err != nil { - return 0, err - } - - return duration, nil -} diff --git a/pkg/exprhelpers/helpers.go b/pkg/exprhelpers/helpers.go index c0163b3e7..4b215cf85 100644 --- a/pkg/exprhelpers/helpers.go +++ b/pkg/exprhelpers/helpers.go @@ -29,6 +29,8 @@ import ( "github.com/umahmood/haversine" "github.com/wasilibs/go-re2" + "github.com/crowdsecurity/go-cs-lib/cstime" + "github.com/crowdsecurity/crowdsec/pkg/cache" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/fflag" @@ -675,7 +677,7 @@ func GetDecisionsSinceCount(params ...any) (any, error) { return 0, nil } - sinceDuration, err := time.ParseDuration(since) + sinceDuration, err := cstime.ParseDurationWithDays(since) if err != nil { log.Errorf("Failed to parse since parameter '%s' : %s", since, err) return 0, nil diff --git a/pkg/parser/stage.go b/pkg/parser/stage.go index ddc07ca7f..22be68fdb 100644 --- a/pkg/parser/stage.go +++ b/pkg/parser/stage.go @@ -37,103 +37,132 @@ type Stagefile struct { } func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx, ectx EnricherCtx) ([]Node, error) { - var nodes []Node - tmpstages := make(map[string]bool) + var allNodes []Node + + tmpStages := make(map[string]bool) pctx.Stages = []string{} - for _, stageFile := range stageFiles { - if !strings.HasSuffix(stageFile.Filename, ".yaml") && !strings.HasSuffix(stageFile.Filename, ".yml") { - log.Warningf("skip non yaml : %s", stageFile.Filename) - continue - } - log.Debugf("loading parser file '%s'", stageFile) - st, err := os.Stat(stageFile.Filename) + for _, sf := range stageFiles { + nodes, err := processStageFile(sf, pctx, ectx) if err != nil { - return nil, fmt.Errorf("failed to stat %s : %v", stageFile, err) + return nil, err } - if st.IsDir() { - continue + + for _, n := range nodes { //nolint:gocritic // rangeValCopy + allNodes = append(allNodes, n) + tmpStages[n.Stage] = true } - yamlFile, err := os.Open(stageFile.Filename) - if err != nil { - return nil, fmt.Errorf("can't access parsing configuration file %s : %s", stageFile.Filename, err) - } - defer yamlFile.Close() - //process the yaml - dec := yaml.NewDecoder(yamlFile) - dec.SetStrict(true) - nodesCount := 0 - for { - node := Node{} - node.OnSuccess = "continue" //default behavior is to continue - err = dec.Decode(&node) - if err != nil { - if errors.Is(err, io.EOF) { - log.Tracef("End of yaml file") - break - } - return nil, fmt.Errorf("error decoding parsing configuration file '%s': %v", stageFile.Filename, err) - } - - //check for empty bucket - if node.Name == "" && node.Description == "" && node.Author == "" { - log.Infof("Node in %s has no name, author or description. Skipping.", stageFile.Filename) - continue - } - //check compat - if node.FormatVersion == "" { - log.Tracef("no version in %s, assuming '1.0'", node.Name) - node.FormatVersion = "1.0" - } - ok, err := constraint.Satisfies(node.FormatVersion, constraint.Parser) - if err != nil { - return nil, fmt.Errorf("failed to check version : %s", err) - } - if !ok { - log.Errorf("%s : %s doesn't satisfy parser format %s, skip", node.Name, node.FormatVersion, constraint.Parser) - continue - } - - node.Stage = stageFile.Stage - if _, ok := tmpstages[stageFile.Stage]; !ok { - tmpstages[stageFile.Stage] = true - } - //compile the node : grok pattern and expression - err = node.compile(pctx, ectx) - if err != nil { - if node.Name != "" { - return nil, fmt.Errorf("failed to compile node '%s' in '%s' : %s", node.Name, stageFile.Filename, err) - } - return nil, fmt.Errorf("failed to compile node in '%s' : %s", stageFile.Filename, err) - } - /* if the stage is empty, the node is empty, it's a trailing entry in users yaml file */ - if node.Stage == "" { - continue - } - - for _, data := range node.Data { - err = exprhelpers.FileInit(pctx.DataFolder, data.DestPath, data.Type) - if err != nil { - log.Error(err.Error()) - } - if data.Type == "regexp" { //cache only makes sense for regexp - if err = exprhelpers.RegexpCacheInit(data.DestPath, *data); err != nil { - log.Error(err.Error()) - } - } - } - - nodes = append(nodes, node) - nodesCount++ - } - log.WithFields(log.Fields{"file": stageFile.Filename, "stage": stageFile.Stage}).Infof("Loaded %d parser nodes", nodesCount) } - for k := range tmpstages { + for k := range tmpStages { pctx.Stages = append(pctx.Stages, k) } + sort.Strings(pctx.Stages) - log.Infof("Loaded %d nodes from %d stages", len(nodes), len(pctx.Stages)) + log.Infof("Loaded %d nodes from %d stages", len(allNodes), len(pctx.Stages)) + + return allNodes, nil +} + +func processStageFile(stageFile Stagefile, pctx *UnixParserCtx, ectx EnricherCtx) ([]Node, error) { + if !strings.HasSuffix(stageFile.Filename, ".yaml") && !strings.HasSuffix(stageFile.Filename, ".yml") { + log.Warningf("skip non yaml : %s", stageFile.Filename) + return nil, nil + } + + log.Debugf("loading parser file '%s'", stageFile) + + st, err := os.Stat(stageFile.Filename) + if err != nil { + return nil, fmt.Errorf("failed to stat %s : %v", stageFile, err) + } + + if st.IsDir() { + return nil, nil + } + + yamlFile, err := os.Open(stageFile.Filename) + if err != nil { + return nil, fmt.Errorf("can't access parsing configuration file %s : %s", stageFile.Filename, err) + } + defer yamlFile.Close() + // process the yaml + dec := yaml.NewDecoder(yamlFile) + dec.SetStrict(true) + + var nodes []Node + + nodesCount := 0 + + for { + node := Node{} + node.OnSuccess = "continue" // default behavior is to continue + + if err = dec.Decode(&node); err != nil { + if errors.Is(err, io.EOF) { + log.Tracef("End of yaml file") + break + } + + return nil, fmt.Errorf("error decoding parsing configuration file '%s': %v", stageFile.Filename, err) + } + + // check for empty bucket + if node.Name == "" && node.Description == "" && node.Author == "" { + log.Infof("Node in %s has no name, author or description. Skipping.", stageFile.Filename) + continue + } + + // check compat + if node.FormatVersion == "" { + log.Tracef("no version in %s, assuming '1.0'", node.Name) + node.FormatVersion = "1.0" + } + + ok, err := constraint.Satisfies(node.FormatVersion, constraint.Parser) + if err != nil { + return nil, fmt.Errorf("failed to check version : %s", err) + } + + if !ok { + log.Errorf("%s : %s doesn't satisfy parser format %s, skip", node.Name, node.FormatVersion, constraint.Parser) + continue + } + + node.Stage = stageFile.Stage + // compile the node : grok pattern and expression + + err = node.compile(pctx, ectx) + if err != nil { + if node.Name != "" { + return nil, fmt.Errorf("failed to compile node '%s' in '%s' : %s", node.Name, stageFile.Filename, err) + } + + return nil, fmt.Errorf("failed to compile node in '%s' : %s", stageFile.Filename, err) + } + /* if the stage is empty, the node is empty, it's a trailing entry in users yaml file */ + if node.Stage == "" { + continue + } + + for _, data := range node.Data { + err = exprhelpers.FileInit(pctx.DataFolder, data.DestPath, data.Type) + if err != nil { + log.Error(err.Error()) + } + + if data.Type == "regexp" { // cache only makes sense for regexp + if err = exprhelpers.RegexpCacheInit(data.DestPath, *data); err != nil { + log.Error(err.Error()) + } + } + } + + nodes = append(nodes, node) + nodesCount++ + } + + log.WithFields(log.Fields{"file": stageFile.Filename, "stage": stageFile.Stage}).Infof("Loaded %d parser nodes", nodesCount) return nodes, nil } diff --git a/test/bats/10_bouncers.bats b/test/bats/10_bouncers.bats index 382205796..9381a4528 100644 --- a/test/bats/10_bouncers.bats +++ b/test/bats/10_bouncers.bats @@ -133,6 +133,13 @@ teardown() { } @test "cscli bouncers prune" { + rune -1 cscli bouncers prune --duration foobar + assert_stderr 'Error: invalid argument "foobar" for "-d, --duration" flag: time: invalid duration "foobar"' + + # duration takes days as well + rune -0 cscli bouncers prune --duration 1d30m + assert_output 'No bouncers to prune.' + rune -0 cscli bouncers prune assert_output 'No bouncers to prune.' rune -0 cscli bouncers add ciTestBouncer diff --git a/test/bats/30_machines.bats b/test/bats/30_machines.bats index 989cd5fe6..97f8584a1 100644 --- a/test/bats/30_machines.bats +++ b/test/bats/30_machines.bats @@ -124,12 +124,19 @@ teardown() { @test "cscli machines prune" { rune -0 cscli metrics + rune -1 cscli machines prune --duration foobar + assert_stderr 'Error: invalid argument "foobar" for "-d, --duration" flag: time: invalid duration "foobar"' + # if the fixture has been created some time ago, # the machines may be old enough to trigger a user prompt. # make sure the prune duration is high enough. rune -0 cscli machines prune --duration 1000000h assert_output 'No machines to prune.' + # duration takes days as well + rune -0 cscli machines prune --duration 1000d30m + assert_output 'No machines to prune.' + rune -0 cscli machines list -o json rune -0 jq -r '.[-1].machineId' <(output) rune -0 cscli machines delete "$output" diff --git a/test/bats/80_alerts.bats b/test/bats/80_alerts.bats index 78c4e67a7..0fadea984 100644 --- a/test/bats/80_alerts.bats +++ b/test/bats/80_alerts.bats @@ -43,6 +43,15 @@ teardown() { assert_output --regexp " githubciXXXXXXXXXXXXXXXXXXXXXXXX([a-zA-Z0-9]{16})? " } +@test "cscli alerts list, accept duration parameters with days" { + rune -1 cscli alerts list --until toto + assert_stderr 'Error: invalid argument "toto" for "--until" flag: time: invalid duration "toto"' + rune -0 cscli alerts list --until 2d12h --debug + assert_stderr --partial "until=60h0m0s" + rune -0 cscli alerts list --since 2d12h --debug + assert_stderr --partial "since=60h0m0s" +} + @test "cscli alerts list, human/json/raw" { rune -0 cscli decisions add -i 10.20.30.40 -t ban diff --git a/test/bats/90_decisions.bats b/test/bats/90_decisions.bats index 549048965..5ba29659e 100644 --- a/test/bats/90_decisions.bats +++ b/test/bats/90_decisions.bats @@ -54,9 +54,13 @@ teardown() { assert_output --regexp " githubciXXXXXXXXXXXXXXXXXXXXXXXX([a-zA-Z0-9]{16})? " } -@test "cscli decisions list, incorrect parameters" { +@test "cscli decisions list, accept duration parameters with days" { rune -1 cscli decisions list --until toto - assert_stderr 'Error: unable to retrieve decisions: performing request: API error: while parsing duration: time: invalid duration "toto"' + assert_stderr 'Error: invalid argument "toto" for "--until" flag: time: invalid duration "toto"' + rune -0 cscli decisions list --until 2d12h --debug + assert_stderr --partial "until=60h0m0s" + rune -0 cscli decisions list --since 2d12h --debug + assert_stderr --partial "since=60h0m0s" } @test "cscli decisions import" { diff --git a/test/bats/cscli-allowlists.bats b/test/bats/cscli-allowlists.bats index 24810110d..d357f29f9 100644 --- a/test/bats/cscli-allowlists.bats +++ b/test/bats/cscli-allowlists.bats @@ -119,14 +119,14 @@ teardown() { # comment and expiration are applied to all values rune -1 cscli allowlist add foo 10.10.10.10 10.20.30.40 -d comment -e toto - assert_stderr 'Error: time: invalid duration "toto"' + assert_stderr 'Error: invalid argument "toto" for "-e, --expiration" flag: time: invalid duration "toto"' refute_output rune -1 cscli allowlist add foo 10.10.10.10 10.20.30.40 -d comment -e '1 day' refute_output - assert_stderr 'Error: strconv.Atoi: parsing "1 ": invalid syntax' + assert_stderr 'Error: invalid argument "1 day" for "-e, --expiration" flag: invalid day value in duration "1 day"' - rune -0 cscli allowlist add foo 10.10.10.10 -d comment -e '1d' + rune -0 cscli allowlist add foo 10.10.10.10 -d comment -e '1d12h' assert_output 'added 1 values to allowlist foo' refute_stderr @@ -246,3 +246,58 @@ teardown() { rune -0 jq 'del(.created_at) | del(.updated_at) | del(.items.[].created_at) | del(.items.[].expiration)' <(output) assert_json '{"description":"a foo","items":[],"name":"foo"}' } + +@test "allowlists expire active decisions" { + rune -0 cscli decisions add -i 1.2.3.4 + rune -0 cscli decisions add -r 2.3.4.0/24 + rune -0 cscli decisions add -i 5.4.3.42 + rune -0 cscli decisions add -r 6.5.4.0/24 + rune -0 cscli decisions add -r 10.0.0.0/23 + + rune -0 cscli decisions list -o json + rune -0 jq -r 'sort_by(.decisions[].value) | .[].decisions[0].value' <(output) + assert_output - <<-EOT + 1.2.3.4 + 10.0.0.0/23 + 2.3.4.0/24 + 5.4.3.42 + 6.5.4.0/24 + EOT + + rune -0 cscli allowlists create foo -d "foo" + + # add an allowlist that matches exactly + rune -0 cscli allowlists add foo 1.2.3.4 + if is_db_mysql; then sleep 2; fi + # it should not be here anymore + rune -0 cscli decisions list -o json + rune -0 jq -e 'any(.[].decisions[]; .value == "1.2.3.4") | not' <(output) + + # allowlist an IP belonging to a range + rune -0 cscli allowlist add foo 2.3.4.42 + if is_db_mysql; then sleep 2; fi + rune -0 cscli decisions list -o json + rune -0 jq -e 'any(.[].decisions[]; .value == "2.3.4.0/24") | not' <(output) + + # allowlist a range with an active decision inside + rune -0 cscli allowlist add foo 5.4.3.0/24 + if is_db_mysql; then sleep 2; fi + rune -0 cscli decisions list -o json + rune -0 jq -e 'any(.[].decisions[]; .value == "5.4.3.42") | not' <(output) + + # allowlist a range inside a range for which we have a decision + rune -0 cscli allowlist add foo 6.5.4.0/25 + if is_db_mysql; then sleep 2; fi + rune -0 cscli decisions list -o json + rune -0 jq -e 'any(.[].decisions[]; .value == "6.5.4.0/24") | not' <(output) + + # allowlist a range bigger than a range for which we have a decision + rune -0 cscli allowlist add foo 10.0.0.0/24 + if is_db_mysql; then sleep 2; fi + rune -0 cscli decisions list -o json + rune -0 jq -e 'any(.[].decisions[]; .value == "10.0.0.0/24") | not' <(output) + + # sanity check no more active decisions + rune -0 cscli decisions list -o json + assert_json [] +} diff --git a/test/lib/config/config-global b/test/lib/config/config-global index 83d95e68e..014b7cc19 100755 --- a/test/lib/config/config-global +++ b/test/lib/config/config-global @@ -116,7 +116,7 @@ load_init_data() { dump_backend="$(cat "${LOCAL_INIT_DIR}/.backend")" if [[ "${DB_BACKEND}" != "${dump_backend}" ]]; then - die "Can't run with backend '${DB_BACKEND}' because the test data was built with '${dump_backend}'" + die "Can't run with backend '${DB_BACKEND}' because 'make bats-fixture' was ran with '${dump_backend}'" fi remove_init_data diff --git a/test/lib/config/config-local b/test/lib/config/config-local index 54ac8550c..e6588c61c 100755 --- a/test/lib/config/config-local +++ b/test/lib/config/config-local @@ -168,7 +168,7 @@ load_init_data() { dump_backend="$(cat "${LOCAL_INIT_DIR}/.backend")" if [[ "${DB_BACKEND}" != "${dump_backend}" ]]; then - die "Can't run with backend '${DB_BACKEND}' because the test data was built with '${dump_backend}'" + die "Can't run with backend '${DB_BACKEND}' because 'make bats-fixture' was ran with '${dump_backend}'" fi remove_init_data diff --git a/test/run-tests b/test/run-tests index e7609188c..ed9d6f553 100755 --- a/test/run-tests +++ b/test/run-tests @@ -26,7 +26,7 @@ fi dump_backend="$(cat "$LOCAL_INIT_DIR/.backend")" if [[ "$DB_BACKEND" != "$dump_backend" ]]; then - die "Can't run with backend '$DB_BACKEND' because the test data was build with '$dump_backend'" + die "Can't run with backend '$DB_BACKEND' because 'make bats-fixture' was ran with '$dump_backend'" fi if [[ $# -ge 1 ]]; then