From f8f0b2a211f05927669ccadaac56ff855dad59ae Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Mon, 5 May 2025 15:12:29 +0200 Subject: [PATCH] improve support for parsing time durations with 'day' units (#3599) * custom duration type for "cscli decisions list", "cscli alerts list" * custom duration type for "cscli allowlist add" * custom duration type for "cscli machines prune" * custom duration type for "cscli bouncers prune" * replace old function ParseDuration * use custom duration type in expr helpers * update dependency * lint * test fix * support days in 'metrics_max_age' * DurationWithDays for 'max_age' --- cmd/crowdsec-cli/clialert/alerts.go | 49 ++++-------------- cmd/crowdsec-cli/cliallowlists/allowlists.go | 17 ++---- cmd/crowdsec-cli/clibouncer/prune.go | 15 +++--- cmd/crowdsec-cli/clidecision/decisions.go | 38 +++----------- cmd/crowdsec-cli/climachine/prune.go | 11 ++-- go.mod | 10 ++-- go.sum | 15 +++--- pkg/apiclient/alerts_service.go | 54 ++++++++++---------- pkg/apiserver/apiserver_test.go | 17 +++--- pkg/csconfig/database.go | 9 ++-- pkg/csprofiles/csprofiles.go | 9 ++-- pkg/database/alertfilter.go | 6 ++- pkg/database/alerts.go | 3 +- pkg/database/flush.go | 33 +++++------- pkg/database/utils.go | 28 ---------- pkg/exprhelpers/helpers.go | 4 +- test/bats/10_bouncers.bats | 7 +++ test/bats/30_machines.bats | 7 +++ test/bats/80_alerts.bats | 9 ++++ test/bats/90_decisions.bats | 8 ++- test/bats/cscli-allowlists.bats | 6 +-- 21 files changed, 153 insertions(+), 202 deletions(-) diff --git a/cmd/crowdsec-cli/clialert/alerts.go b/cmd/crowdsec-cli/clialert/alerts.go index 3e69acfec..88e870ee7 100644 --- a/cmd/crowdsec-cli/clialert/alerts.go +++ b/cmd/crowdsec-cli/clialert/alerts.go @@ -12,6 +12,7 @@ import ( "strconv" "strings" "text/template" + "time" "github.com/fatih/color" "github.com/go-openapi/strfmt" @@ -19,6 +20,7 @@ import ( "github.com/spf13/cobra" "gopkg.in/yaml.v3" + "github.com/crowdsecurity/go-cs-lib/cstime" "github.com/crowdsecurity/go-cs-lib/maptools" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/args" @@ -247,34 +249,6 @@ func (cli *cliAlerts) list(ctx context.Context, alertListFilter apiclient.Alerts alertListFilter.Limit = limit } - if *alertListFilter.Until == "" { - alertListFilter.Until = nil - } else if strings.HasSuffix(*alertListFilter.Until, "d") { - /*time.ParseDuration support hours 'h' as bigger unit, let's make the user's life easier*/ - realDuration := strings.TrimSuffix(*alertListFilter.Until, "d") - - days, err := strconv.Atoi(realDuration) - if err != nil { - return fmt.Errorf("can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *alertListFilter.Until) - } - - *alertListFilter.Until = fmt.Sprintf("%d%s", days*24, "h") - } - - if *alertListFilter.Since == "" { - alertListFilter.Since = nil - } else if strings.HasSuffix(*alertListFilter.Since, "d") { - // time.ParseDuration support hours 'h' as bigger unit, let's make the user's life easier - realDuration := strings.TrimSuffix(*alertListFilter.Since, "d") - - days, err := strconv.Atoi(realDuration) - if err != nil { - return fmt.Errorf("can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *alertListFilter.Since) - } - - *alertListFilter.Since = fmt.Sprintf("%d%s", days*24, "h") - } - if *alertListFilter.IncludeCAPI { *alertListFilter.Limit = 0 } @@ -330,8 +304,8 @@ func (cli *cliAlerts) newListCmd() *cobra.Command { ScenarioEquals: new(string), IPEquals: new(string), RangeEquals: new(string), - Since: new(string), - Until: new(string), + Since: cstime.DurationWithDays(0), + Until: cstime.DurationWithDays(0), TypeEquals: new(string), IncludeCAPI: new(bool), OriginEquals: new(string), @@ -362,8 +336,8 @@ cscli alerts list --type ban`, flags := cmd.Flags() flags.SortFlags = false flags.BoolVarP(alertListFilter.IncludeCAPI, "all", "a", false, "Include decisions from Central API") - flags.StringVar(alertListFilter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)") - flags.StringVar(alertListFilter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)") + flags.Var(&alertListFilter.Until, "until", "restrict to alerts older than until (ie. 4h, 30d)") + flags.Var(&alertListFilter.Since, "since", "restrict to alerts newer than since (ie. 4h, 30d)") flags.StringVarP(alertListFilter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value )") flags.StringVarP(alertListFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)") flags.StringVarP(alertListFilter.RangeEquals, "range", "r", "", "restrict to alerts from this range (shorthand for --scope range --value )") @@ -560,10 +534,9 @@ func (cli *cliAlerts) newInspectCmd() *cobra.Command { } func (cli *cliAlerts) newFlushCmd() *cobra.Command { - var ( - maxItems int - maxAge string - ) + var maxItems int + + maxAge := cstime.DurationWithDays(7*24*time.Hour) cmd := &cobra.Command{ Use: `flush`, @@ -584,7 +557,7 @@ func (cli *cliAlerts) newFlushCmd() *cobra.Command { return err } log.Info("Flushing alerts. !! This may take a long time !!") - err = db.FlushAlerts(ctx, maxAge, maxItems) + err = db.FlushAlerts(ctx, time.Duration(maxAge), maxItems) if err != nil { return fmt.Errorf("unable to flush alerts: %w", err) } @@ -596,7 +569,7 @@ func (cli *cliAlerts) newFlushCmd() *cobra.Command { cmd.Flags().SortFlags = false cmd.Flags().IntVar(&maxItems, "max-items", 5000, "Maximum number of alert items to keep in the database") - cmd.Flags().StringVar(&maxAge, "max-age", "7d", "Maximum age of alert items to keep in the database") + cmd.Flags().Var(&maxAge, "max-age", "Maximum age of alert items to keep in the database") return cmd } diff --git a/cmd/crowdsec-cli/cliallowlists/allowlists.go b/cmd/crowdsec-cli/cliallowlists/allowlists.go index be4b966be..972ee1b47 100644 --- a/cmd/crowdsec-cli/cliallowlists/allowlists.go +++ b/cmd/crowdsec-cli/cliallowlists/allowlists.go @@ -399,8 +399,8 @@ func (cli *cliAllowLists) delete(ctx context.Context, db *database.Client, name func (cli *cliAllowLists) newAddCmd() *cobra.Command { var ( - expirationStr string - comment string + expiration cstime.DurationWithDays + comment string ) cmd := &cobra.Command{ @@ -424,25 +424,16 @@ func (cli *cliAllowLists) newAddCmd() *cobra.Command { return err } - var expiration time.Duration - - if expirationStr != "" { - expiration, err = cstime.ParseDuration(expirationStr) - if err != nil { - return err - } - } - name := args[0] values := args[1:] - return cli.add(ctx, db, name, values, expiration, comment) + return cli.add(ctx, db, name, values, time.Duration(expiration), comment) }, } flags := cmd.Flags() - flags.StringVarP(&expirationStr, "expiration", "e", "", "expiration duration") + flags.VarP(&expiration, "expiration", "e", "expiration duration") flags.StringVarP(&comment, "comment", "d", "", "comment for the value") return cmd diff --git a/cmd/crowdsec-cli/clibouncer/prune.go b/cmd/crowdsec-cli/clibouncer/prune.go index 3c27efe39..cf70e63f4 100644 --- a/cmd/crowdsec-cli/clibouncer/prune.go +++ b/cmd/crowdsec-cli/clibouncer/prune.go @@ -9,10 +9,14 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" + "github.com/crowdsecurity/go-cs-lib/cstime" + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/args" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/ask" ) +const defaultPruneDuration = 60 * time.Minute + func (cli *cliBouncers) prune(ctx context.Context, duration time.Duration, force bool) error { if duration < 2*time.Minute { if yes, err := ask.YesNo( @@ -59,12 +63,9 @@ func (cli *cliBouncers) prune(ctx context.Context, duration time.Duration, force } func (cli *cliBouncers) newPruneCmd() *cobra.Command { - var ( - duration time.Duration - force bool - ) + var force bool - const defaultDuration = 60 * time.Minute + duration := cstime.DurationWithDays(defaultPruneDuration) cmd := &cobra.Command{ Use: "prune", @@ -74,12 +75,12 @@ func (cli *cliBouncers) newPruneCmd() *cobra.Command { Example: `cscli bouncers prune -d 45m cscli bouncers prune -d 45m --force`, RunE: func(cmd *cobra.Command, _ []string) error { - return cli.prune(cmd.Context(), duration, force) + return cli.prune(cmd.Context(), time.Duration(duration), force) }, } flags := cmd.Flags() - flags.DurationVarP(&duration, "duration", "d", defaultDuration, "duration of time since last pull") + flags.VarP(&duration, "duration", "d", "duration of time since last pull") flags.BoolVar(&force, "force", false, "force prune without asking for confirmation") return cmd diff --git a/cmd/crowdsec-cli/clidecision/decisions.go b/cmd/crowdsec-cli/clidecision/decisions.go index 91f39f421..c141f1287 100644 --- a/cmd/crowdsec-cli/clidecision/decisions.go +++ b/cmd/crowdsec-cli/clidecision/decisions.go @@ -23,6 +23,8 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" + + "github.com/crowdsecurity/go-cs-lib/cstime" ) type configGetter func() *csconfig.Config @@ -184,34 +186,8 @@ func (cli *cliDecisions) list(ctx context.Context, filter apiclient.AlertsListOp if noSimu != nil && *noSimu { filter.IncludeSimulated = new(bool) } + /* nullify the empty entries to avoid bad filter */ - if *filter.Until == "" { - filter.Until = nil - } else if strings.HasSuffix(*filter.Until, "d") { - /*time.ParseDuration support hours 'h' as bigger unit, let's make the user's life easier*/ - realDuration := strings.TrimSuffix(*filter.Until, "d") - - days, err := strconv.Atoi(realDuration) - if err != nil { - return fmt.Errorf("can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until) - } - - *filter.Until = fmt.Sprintf("%d%s", days*24, "h") - } - - if *filter.Since == "" { - filter.Since = nil - } else if strings.HasSuffix(*filter.Since, "d") { - /*time.ParseDuration support hours 'h' as bigger unit, let's make the user's life easier*/ - realDuration := strings.TrimSuffix(*filter.Since, "d") - - days, err := strconv.Atoi(realDuration) - if err != nil { - return fmt.Errorf("can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Since) - } - - *filter.Since = fmt.Sprintf("%d%s", days*24, "h") - } if *filter.IncludeCAPI { *filter.Limit = 0 @@ -270,8 +246,8 @@ func (cli *cliDecisions) newListCmd() *cobra.Command { OriginEquals: new(string), IPEquals: new(string), RangeEquals: new(string), - Since: new(string), - Until: new(string), + Since: cstime.DurationWithDays(0), + Until: cstime.DurationWithDays(0), TypeEquals: new(string), IncludeCAPI: new(bool), Limit: new(int), @@ -300,8 +276,8 @@ cscli decisions list --origin lists --scenario list_name flags := cmd.Flags() flags.SortFlags = false flags.BoolVarP(filter.IncludeCAPI, "all", "a", false, "Include decisions from Central API") - flags.StringVar(filter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)") - flags.StringVar(filter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)") + flags.Var(&filter.Since, "since", "restrict to alerts newer than since (ie. 4h, 30d)") + flags.Var(&filter.Until, "until", "restrict to alerts older than until (ie. 4h, 30d)") flags.StringVarP(filter.TypeEquals, "type", "t", "", "restrict to this decision type (ie. ban,captcha)") flags.StringVar(filter.ScopeEquals, "scope", "", "restrict to this scope (ie. ip,range,session)") flags.StringVar(filter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ","))) diff --git a/cmd/crowdsec-cli/climachine/prune.go b/cmd/crowdsec-cli/climachine/prune.go index 4054305f4..be9102428 100644 --- a/cmd/crowdsec-cli/climachine/prune.go +++ b/cmd/crowdsec-cli/climachine/prune.go @@ -9,11 +9,15 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" + "github.com/crowdsecurity/go-cs-lib/cstime" + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/args" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/ask" "github.com/crowdsecurity/crowdsec/pkg/database/ent" ) +const defaultPruneDuration = 10 * time.Minute + func (cli *cliMachines) prune(ctx context.Context, duration time.Duration, notValidOnly bool, force bool) error { if duration < 2*time.Minute && !notValidOnly { if yes, err := ask.YesNo( @@ -67,12 +71,11 @@ func (cli *cliMachines) prune(ctx context.Context, duration time.Duration, notVa func (cli *cliMachines) newPruneCmd() *cobra.Command { var ( - duration time.Duration notValidOnly bool force bool ) - const defaultDuration = 10 * time.Minute + duration := cstime.DurationWithDays(defaultPruneDuration) cmd := &cobra.Command{ Use: "prune", @@ -84,12 +87,12 @@ cscli machines prune --not-validated-only --force`, Args: args.NoArgs, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, _ []string) error { - return cli.prune(cmd.Context(), duration, notValidOnly, force) + return cli.prune(cmd.Context(), time.Duration(duration), notValidOnly, force) }, } flags := cmd.Flags() - flags.DurationVarP(&duration, "duration", "d", defaultDuration, "duration of time since validated machine last heartbeat") + flags.VarP(&duration, "duration", "d", "duration of time since validated machine last heartbeat") flags.BoolVar(¬ValidOnly, "not-validated-only", false, "only prune machines that are not validated") flags.BoolVar(&force, "force", false, "force prune without asking for confirmation") diff --git a/go.mod b/go.mod index 81c41acc9..6c027fec4 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/creack/pty v1.1.21 // indirect github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 - github.com/crowdsecurity/go-cs-lib v0.0.18 + github.com/crowdsecurity/go-cs-lib v0.0.19 github.com/crowdsecurity/grokky v0.2.2 github.com/crowdsecurity/machineid v1.0.2 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc @@ -87,8 +87,8 @@ require ( github.com/shirou/gopsutil/v3 v3.23.5 github.com/sirupsen/logrus v1.9.3 github.com/slack-go/slack v0.16.0 - github.com/spf13/cobra v1.8.1 - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/cobra v1.9.1 + github.com/spf13/pflag v1.0.6 // indirect github.com/stretchr/testify v1.10.0 github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 github.com/wasilibs/go-re2 v1.7.0 @@ -100,7 +100,7 @@ require ( go.opentelemetry.io/otel/trace v1.28.0 // indirect golang.org/x/crypto v0.36.0 golang.org/x/mod v0.23.0 - golang.org/x/net v0.38.0 // indirect + golang.org/x/net v0.38.0 golang.org/x/sync v0.12.0 golang.org/x/sys v0.31.0 golang.org/x/text v0.23.0 @@ -131,7 +131,7 @@ require ( github.com/bytedance/sonic/loader v0.2.1 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gabriel-vasile/mimetype v1.4.7 // indirect github.com/gin-contrib/sse v0.1.0 // indirect diff --git a/go.sum b/go.sum index 133e1e9e0..ece7d98e1 100644 --- a/go.sum +++ b/go.sum @@ -100,8 +100,8 @@ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7 github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= @@ -111,8 +111,8 @@ github.com/crowdsecurity/coraza/v3 v3.0.0-20250320231801-749b8bded21a h1:2Nyr+47 github.com/crowdsecurity/coraza/v3 v3.0.0-20250320231801-749b8bded21a/go.mod h1:xSaXWOhFMSbrV8qOOfBKAyw3aOqfwaSaOy5BgSF8XlA= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= -github.com/crowdsecurity/go-cs-lib v0.0.18 h1:GNyvaag5MXfuapIy4E30pIOvIE5AyHoanJBNSMA1cmE= -github.com/crowdsecurity/go-cs-lib v0.0.18/go.mod h1:XwGcvTt4lMq4Tm1IRMSKMDf0CVrnytTU8Uoofa7AR+g= +github.com/crowdsecurity/go-cs-lib v0.0.19 h1:wA4O8hGrEntTGn7eZTJqnQ3mrAje5JvQAj8DNbe5IZg= +github.com/crowdsecurity/go-cs-lib v0.0.19/go.mod h1:hz2FOHFXc0vWzH78uxo2VebtPQ9Snkbdzy3TMA20tVQ= github.com/crowdsecurity/grokky v0.2.2 h1:yALsI9zqpDArYzmSSxfBq2dhYuGUTKMJq8KOEIAsuo4= github.com/crowdsecurity/grokky v0.2.2/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM= github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc= @@ -660,11 +660,12 @@ github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= diff --git a/pkg/apiclient/alerts_service.go b/pkg/apiclient/alerts_service.go index 1f84862a8..b44049af4 100644 --- a/pkg/apiclient/alerts_service.go +++ b/pkg/apiclient/alerts_service.go @@ -7,42 +7,44 @@ import ( qs "github.com/google/go-querystring/query" + "github.com/crowdsecurity/go-cs-lib/cstime" + "github.com/crowdsecurity/crowdsec/pkg/models" ) type AlertsService service type AlertsListOpts struct { - ScopeEquals *string `url:"scope,omitempty"` - ValueEquals *string `url:"value,omitempty"` - ScenarioEquals *string `url:"scenario,omitempty"` - IPEquals *string `url:"ip,omitempty"` - RangeEquals *string `url:"range,omitempty"` - OriginEquals *string `url:"origin,omitempty"` - Since *string `url:"since,omitempty"` - TypeEquals *string `url:"decision_type,omitempty"` - Until *string `url:"until,omitempty"` - IncludeSimulated *bool `url:"simulated,omitempty"` - ActiveDecisionEquals *bool `url:"has_active_decision,omitempty"` - IncludeCAPI *bool `url:"include_capi,omitempty"` - Limit *int `url:"limit,omitempty"` - Contains *bool `url:"contains,omitempty"` + ScopeEquals *string `url:"scope,omitempty"` + ValueEquals *string `url:"value,omitempty"` + ScenarioEquals *string `url:"scenario,omitempty"` + IPEquals *string `url:"ip,omitempty"` + RangeEquals *string `url:"range,omitempty"` + OriginEquals *string `url:"origin,omitempty"` + Since cstime.DurationWithDays `url:"since,omitempty"` + TypeEquals *string `url:"decision_type,omitempty"` + Until cstime.DurationWithDays `url:"until,omitempty"` + IncludeSimulated *bool `url:"simulated,omitempty"` + ActiveDecisionEquals *bool `url:"has_active_decision,omitempty"` + IncludeCAPI *bool `url:"include_capi,omitempty"` + Limit *int `url:"limit,omitempty"` + Contains *bool `url:"contains,omitempty"` ListOpts } type AlertsDeleteOpts struct { - ScopeEquals *string `url:"scope,omitempty"` - ValueEquals *string `url:"value,omitempty"` - ScenarioEquals *string `url:"scenario,omitempty"` - IPEquals *string `url:"ip,omitempty"` - RangeEquals *string `url:"range,omitempty"` - Since *string `url:"since,omitempty"` - Until *string `url:"until,omitempty"` - OriginEquals *string `url:"origin,omitempty"` - ActiveDecisionEquals *bool `url:"has_active_decision,omitempty"` - SourceEquals *string `url:"alert_source,omitempty"` - Contains *bool `url:"contains,omitempty"` - Limit *int `url:"limit,omitempty"` + ScopeEquals *string `url:"scope,omitempty"` + ValueEquals *string `url:"value,omitempty"` + ScenarioEquals *string `url:"scenario,omitempty"` + IPEquals *string `url:"ip,omitempty"` + RangeEquals *string `url:"range,omitempty"` + Since cstime.DurationWithDays `url:"since,omitempty"` + Until cstime.DurationWithDays `url:"until,omitempty"` + OriginEquals *string `url:"origin,omitempty"` + ActiveDecisionEquals *bool `url:"has_active_decision,omitempty"` + SourceEquals *string `url:"alert_source,omitempty"` + Contains *bool `url:"contains,omitempty"` + Limit *int `url:"limit,omitempty"` ListOpts } diff --git a/pkg/apiserver/apiserver_test.go b/pkg/apiserver/apiserver_test.go index 01a8588df..6cb0e3d54 100644 --- a/pkg/apiserver/apiserver_test.go +++ b/pkg/apiserver/apiserver_test.go @@ -18,6 +18,7 @@ import ( "github.com/stretchr/testify/require" "github.com/crowdsecurity/go-cs-lib/cstest" + "github.com/crowdsecurity/go-cs-lib/cstime" "github.com/crowdsecurity/go-cs-lib/ptr" "github.com/crowdsecurity/go-cs-lib/version" @@ -47,9 +48,9 @@ var ( func LoadTestConfig(t *testing.T) csconfig.Config { config := csconfig.Config{} - maxAge := "1h" + maxAge := cstime.DurationWithDays(1*time.Hour) flushConfig := csconfig.FlushDBCfg{ - MaxAge: &maxAge, + MaxAge: maxAge, } tempDir, _ := os.MkdirTemp("", "crowdsec_tests") @@ -97,9 +98,9 @@ func LoadTestConfig(t *testing.T) csconfig.Config { func LoadTestConfigForwardedFor(t *testing.T) csconfig.Config { config := csconfig.Config{} - maxAge := "1h" + maxAge := cstime.DurationWithDays(1*time.Hour) flushConfig := csconfig.FlushDBCfg{ - MaxAge: &maxAge, + MaxAge: maxAge, } tempDir, _ := os.MkdirTemp("", "crowdsec_tests") @@ -363,9 +364,9 @@ func TestLoggingDebugToFileConfig(t *testing.T) { ctx := t.Context() /*declare settings*/ - maxAge := "1h" + maxAge := cstime.DurationWithDays(1*time.Hour) flushConfig := csconfig.FlushDBCfg{ - MaxAge: &maxAge, + MaxAge: maxAge, } tempDir, _ := os.MkdirTemp("", "crowdsec_tests") @@ -416,9 +417,9 @@ func TestLoggingErrorToFileConfig(t *testing.T) { ctx := t.Context() /*declare settings*/ - maxAge := "1h" + maxAge := cstime.DurationWithDays(1*time.Hour) flushConfig := csconfig.FlushDBCfg{ - MaxAge: &maxAge, + MaxAge: maxAge, } tempDir, _ := os.MkdirTemp("", "crowdsec_tests") diff --git a/pkg/csconfig/database.go b/pkg/csconfig/database.go index 26150eb2e..f34b40889 100644 --- a/pkg/csconfig/database.go +++ b/pkg/csconfig/database.go @@ -14,6 +14,7 @@ import ( "github.com/go-sql-driver/mysql" log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/go-cs-lib/cstime" "github.com/crowdsecurity/go-cs-lib/ptr" "github.com/crowdsecurity/crowdsec/pkg/types" @@ -58,10 +59,10 @@ type AuthGCCfg struct { type FlushDBCfg struct { MaxItems *int `yaml:"max_items,omitempty"` // We could unmarshal as time.Duration, but alert filters right now are a map of strings - MaxAge *string `yaml:"max_age,omitempty"` - BouncersGC *AuthGCCfg `yaml:"bouncers_autodelete,omitempty"` - AgentsGC *AuthGCCfg `yaml:"agents_autodelete,omitempty"` - MetricsMaxAge *time.Duration `yaml:"metrics_max_age,omitempty"` + MaxAge cstime.DurationWithDays `yaml:"max_age,omitempty"` + BouncersGC *AuthGCCfg `yaml:"bouncers_autodelete,omitempty"` + AgentsGC *AuthGCCfg `yaml:"agents_autodelete,omitempty"` + MetricsMaxAge cstime.DurationWithDays `yaml:"metrics_max_age,omitempty"` } func (c *Config) LoadDBConfig(inCli bool) error { diff --git a/pkg/csprofiles/csprofiles.go b/pkg/csprofiles/csprofiles.go index 0b2ae0280..3081f4d31 100644 --- a/pkg/csprofiles/csprofiles.go +++ b/pkg/csprofiles/csprofiles.go @@ -7,12 +7,13 @@ import ( "github.com/expr-lang/expr/vm" log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/go-cs-lib/cstime" + "github.com/crowdsecurity/go-cs-lib/ptr" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" - utils "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/crowdsecurity/go-cs-lib/ptr" ) type Runtime struct { @@ -84,7 +85,7 @@ func NewProfile(profilesCfg []*csconfig.ProfileCfg) ([]*Runtime, error) { duration = defaultDuration } - if _, err := utils.ParseDuration(duration); err != nil { + if _, err := cstime.ParseDurationWithDays(duration); err != nil { return nil, fmt.Errorf("error parsing duration '%s' of %s: %w", duration, profile.Name, err) } } @@ -136,7 +137,7 @@ func (profile *Runtime) GenerateDecisionFromProfile(alert *models.Alert) ([]*mod profile.Logger.Warningf("Failed to run duration_expr : %v", err) } else { durationStr := fmt.Sprint(duration) - if _, err := utils.ParseDuration(durationStr); err != nil { + if _, err := cstime.ParseDurationWithDays(durationStr); err != nil { profile.Logger.Warningf("Failed to parse expr duration result '%s'", duration) } else { *decision.Duration = durationStr diff --git a/pkg/database/alertfilter.go b/pkg/database/alertfilter.go index 6ff2ab99a..d96690140 100644 --- a/pkg/database/alertfilter.go +++ b/pkg/database/alertfilter.go @@ -9,6 +9,8 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/go-cs-lib/cstime" + "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/alert" "github.com/crowdsecurity/crowdsec/pkg/database/ent/decision" @@ -40,7 +42,9 @@ func handleScopeFilter(scope string, predicates *[]predicate.Alert) { } func handleTimeFilters(param, value string, predicates *[]predicate.Alert) error { - duration, err := ParseDuration(value) + // crowsdec now always sends duration without days, but we allow them for + // compatibility with other tools + duration, err := cstime.ParseDurationWithDays(value) if err != nil { return fmt.Errorf("while parsing duration: %w", err) } diff --git a/pkg/database/alerts.go b/pkg/database/alerts.go index 6f27b5afe..2fcbb8a5f 100644 --- a/pkg/database/alerts.go +++ b/pkg/database/alerts.go @@ -13,6 +13,7 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/go-cs-lib/cstime" "github.com/crowdsecurity/go-cs-lib/slicetools" "github.com/crowdsecurity/crowdsec/pkg/database/ent" @@ -382,7 +383,7 @@ func (c *Client) createDecisionChunk(ctx context.Context, simulated bool, stopAt sz int ) - duration, err := ParseDuration(*decisionItem.Duration) + duration, err := cstime.ParseDurationWithDays(*decisionItem.Duration) if err != nil { return nil, errors.Wrapf(ParseDurationFail, "decision duration '%+v' : %s", *decisionItem.Duration, err) } diff --git a/pkg/database/flush.go b/pkg/database/flush.go index e1b5f9f44..e508c53c0 100644 --- a/pkg/database/flush.go +++ b/pkg/database/flush.go @@ -9,7 +9,7 @@ import ( "github.com/go-co-op/gocron" log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/go-cs-lib/ptr" + "github.com/crowdsecurity/go-cs-lib/cstime" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/database/ent/alert" @@ -30,7 +30,6 @@ const ( func (c *Client) StartFlushScheduler(ctx context.Context, config *csconfig.FlushDBCfg) (*gocron.Scheduler, error) { maxItems := 0 - maxAge := "" if config.MaxItems != nil && *config.MaxItems <= 0 { return nil, errors.New("max_items can't be zero or negative") @@ -40,14 +39,10 @@ func (c *Client) StartFlushScheduler(ctx context.Context, config *csconfig.Flush maxItems = *config.MaxItems } - if config.MaxAge != nil && *config.MaxAge != "" { - maxAge = *config.MaxAge - } - // Init & Start cronjob every minute for alerts scheduler := gocron.NewScheduler(time.UTC) - job, err := scheduler.Every(1).Minute().Do(c.FlushAlerts, ctx, maxAge, maxItems) + job, err := scheduler.Every(1).Minute().Do(c.FlushAlerts, ctx, time.Duration(config.MaxAge), maxItems) if err != nil { return nil, fmt.Errorf("while starting FlushAlerts scheduler: %w", err) } @@ -56,7 +51,7 @@ func (c *Client) StartFlushScheduler(ctx context.Context, config *csconfig.Flush // Init & Start cronjob every hour for bouncers/agents if config.AgentsGC != nil { if config.AgentsGC.Cert != nil { - duration, err := ParseDuration(*config.AgentsGC.Cert) + duration, err := cstime.ParseDurationWithDays(*config.AgentsGC.Cert) if err != nil { return nil, fmt.Errorf("while parsing agents cert auto-delete duration: %w", err) } @@ -65,7 +60,7 @@ func (c *Client) StartFlushScheduler(ctx context.Context, config *csconfig.Flush } if config.AgentsGC.LoginPassword != nil { - duration, err := ParseDuration(*config.AgentsGC.LoginPassword) + duration, err := cstime.ParseDurationWithDays(*config.AgentsGC.LoginPassword) if err != nil { return nil, fmt.Errorf("while parsing agents login/password auto-delete duration: %w", err) } @@ -80,7 +75,7 @@ func (c *Client) StartFlushScheduler(ctx context.Context, config *csconfig.Flush if config.BouncersGC != nil { if config.BouncersGC.Cert != nil { - duration, err := ParseDuration(*config.BouncersGC.Cert) + duration, err := cstime.ParseDurationWithDays(*config.BouncersGC.Cert) if err != nil { return nil, fmt.Errorf("while parsing bouncers cert auto-delete duration: %w", err) } @@ -89,7 +84,7 @@ func (c *Client) StartFlushScheduler(ctx context.Context, config *csconfig.Flush } if config.BouncersGC.Api != nil { - duration, err := ParseDuration(*config.BouncersGC.Api) + duration, err := cstime.ParseDurationWithDays(*config.BouncersGC.Api) if err != nil { return nil, fmt.Errorf("while parsing bouncers api auto-delete duration: %w", err) } @@ -109,7 +104,7 @@ func (c *Client) StartFlushScheduler(ctx context.Context, config *csconfig.Flush baJob.SingletonMode() - metricsJob, err := scheduler.Every(flushInterval).Do(c.flushMetrics, ctx, config.MetricsMaxAge) + metricsJob, err := scheduler.Every(flushInterval).Do(c.flushMetrics, ctx, time.Duration(config.MetricsMaxAge)) if err != nil { return nil, fmt.Errorf("while starting flushMetrics scheduler: %w", err) } @@ -129,15 +124,15 @@ func (c *Client) StartFlushScheduler(ctx context.Context, config *csconfig.Flush } // flushMetrics deletes metrics older than maxAge, regardless if they have been pushed to CAPI or not -func (c *Client) flushMetrics(ctx context.Context, maxAge *time.Duration) { - if maxAge == nil { - maxAge = ptr.Of(defaultMetricsMaxAge) +func (c *Client) flushMetrics(ctx context.Context, maxAge time.Duration) { + if maxAge == 0 { + maxAge = defaultMetricsMaxAge } c.Log.Debugf("flushing metrics older than %s", maxAge) deleted, err := c.Ent.Metric.Delete().Where( - metric.ReceivedAtLTE(time.Now().UTC().Add(-*maxAge)), + metric.ReceivedAtLTE(time.Now().UTC().Add(-maxAge)), ).Exec(ctx) if err != nil { c.Log.Errorf("while flushing metrics: %s", err) @@ -230,7 +225,7 @@ func (c *Client) FlushAgentsAndBouncers(ctx context.Context, agentsCfg *csconfig return nil } -func (c *Client) FlushAlerts(ctx context.Context, maxAge string, maxItems int) error { +func (c *Client) FlushAlerts(ctx context.Context, maxAge time.Duration, maxItems int) error { var ( deletedByAge int deletedByNbItem int @@ -255,9 +250,9 @@ func (c *Client) FlushAlerts(ctx context.Context, maxAge string, maxItems int) e c.Log.Debugf("FlushAlerts (Total alerts): %d", totalAlerts) - if maxAge != "" { + if maxAge != 0 { filter := map[string][]string{ - "created_before": {maxAge}, + "created_before": {maxAge.String()}, } nbDeleted, err := c.DeleteAlertWithFilter(ctx, filter) diff --git a/pkg/database/utils.go b/pkg/database/utils.go index 8148df56f..9b8d20dcf 100644 --- a/pkg/database/utils.go +++ b/pkg/database/utils.go @@ -4,9 +4,6 @@ import ( "encoding/binary" "fmt" "net" - "strconv" - "strings" - "time" ) func IP2Int(ip net.IP) uint32 { @@ -69,28 +66,3 @@ func GetIpsFromIpRange(host string) (int64, int64, error) { return ipStart, ipEnd, nil } - -func ParseDuration(d string) (time.Duration, error) { - durationStr := d - - if strings.HasSuffix(d, "d") { - days := strings.Split(d, "d")[0] - if days == "" { - return 0, fmt.Errorf("'%s' can't be parsed as duration", d) - } - - daysInt, err := strconv.Atoi(days) - if err != nil { - return 0, err - } - - durationStr = strconv.Itoa(daysInt*24) + "h" - } - - duration, err := time.ParseDuration(durationStr) - if err != nil { - return 0, err - } - - return duration, nil -} diff --git a/pkg/exprhelpers/helpers.go b/pkg/exprhelpers/helpers.go index 22bca7d06..6c99c53dd 100644 --- a/pkg/exprhelpers/helpers.go +++ b/pkg/exprhelpers/helpers.go @@ -29,6 +29,8 @@ import ( "github.com/umahmood/haversine" "github.com/wasilibs/go-re2" + "github.com/crowdsecurity/go-cs-lib/cstime" + "github.com/crowdsecurity/crowdsec/pkg/cache" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/fflag" @@ -661,7 +663,7 @@ func GetDecisionsSinceCount(params ...any) (any, error) { return 0, nil } - sinceDuration, err := time.ParseDuration(since) + sinceDuration, err := cstime.ParseDurationWithDays(since) if err != nil { log.Errorf("Failed to parse since parameter '%s' : %s", since, err) return 0, nil diff --git a/test/bats/10_bouncers.bats b/test/bats/10_bouncers.bats index 382205796..9381a4528 100644 --- a/test/bats/10_bouncers.bats +++ b/test/bats/10_bouncers.bats @@ -133,6 +133,13 @@ teardown() { } @test "cscli bouncers prune" { + rune -1 cscli bouncers prune --duration foobar + assert_stderr 'Error: invalid argument "foobar" for "-d, --duration" flag: time: invalid duration "foobar"' + + # duration takes days as well + rune -0 cscli bouncers prune --duration 1d30m + assert_output 'No bouncers to prune.' + rune -0 cscli bouncers prune assert_output 'No bouncers to prune.' rune -0 cscli bouncers add ciTestBouncer diff --git a/test/bats/30_machines.bats b/test/bats/30_machines.bats index 989cd5fe6..97f8584a1 100644 --- a/test/bats/30_machines.bats +++ b/test/bats/30_machines.bats @@ -124,12 +124,19 @@ teardown() { @test "cscli machines prune" { rune -0 cscli metrics + rune -1 cscli machines prune --duration foobar + assert_stderr 'Error: invalid argument "foobar" for "-d, --duration" flag: time: invalid duration "foobar"' + # if the fixture has been created some time ago, # the machines may be old enough to trigger a user prompt. # make sure the prune duration is high enough. rune -0 cscli machines prune --duration 1000000h assert_output 'No machines to prune.' + # duration takes days as well + rune -0 cscli machines prune --duration 1000d30m + assert_output 'No machines to prune.' + rune -0 cscli machines list -o json rune -0 jq -r '.[-1].machineId' <(output) rune -0 cscli machines delete "$output" diff --git a/test/bats/80_alerts.bats b/test/bats/80_alerts.bats index 78c4e67a7..0fadea984 100644 --- a/test/bats/80_alerts.bats +++ b/test/bats/80_alerts.bats @@ -43,6 +43,15 @@ teardown() { assert_output --regexp " githubciXXXXXXXXXXXXXXXXXXXXXXXX([a-zA-Z0-9]{16})? " } +@test "cscli alerts list, accept duration parameters with days" { + rune -1 cscli alerts list --until toto + assert_stderr 'Error: invalid argument "toto" for "--until" flag: time: invalid duration "toto"' + rune -0 cscli alerts list --until 2d12h --debug + assert_stderr --partial "until=60h0m0s" + rune -0 cscli alerts list --since 2d12h --debug + assert_stderr --partial "since=60h0m0s" +} + @test "cscli alerts list, human/json/raw" { rune -0 cscli decisions add -i 10.20.30.40 -t ban diff --git a/test/bats/90_decisions.bats b/test/bats/90_decisions.bats index 549048965..5ba29659e 100644 --- a/test/bats/90_decisions.bats +++ b/test/bats/90_decisions.bats @@ -54,9 +54,13 @@ teardown() { assert_output --regexp " githubciXXXXXXXXXXXXXXXXXXXXXXXX([a-zA-Z0-9]{16})? " } -@test "cscli decisions list, incorrect parameters" { +@test "cscli decisions list, accept duration parameters with days" { rune -1 cscli decisions list --until toto - assert_stderr 'Error: unable to retrieve decisions: performing request: API error: while parsing duration: time: invalid duration "toto"' + assert_stderr 'Error: invalid argument "toto" for "--until" flag: time: invalid duration "toto"' + rune -0 cscli decisions list --until 2d12h --debug + assert_stderr --partial "until=60h0m0s" + rune -0 cscli decisions list --since 2d12h --debug + assert_stderr --partial "since=60h0m0s" } @test "cscli decisions import" { diff --git a/test/bats/cscli-allowlists.bats b/test/bats/cscli-allowlists.bats index 24810110d..e146e8750 100644 --- a/test/bats/cscli-allowlists.bats +++ b/test/bats/cscli-allowlists.bats @@ -119,14 +119,14 @@ teardown() { # comment and expiration are applied to all values rune -1 cscli allowlist add foo 10.10.10.10 10.20.30.40 -d comment -e toto - assert_stderr 'Error: time: invalid duration "toto"' + assert_stderr 'Error: invalid argument "toto" for "-e, --expiration" flag: time: invalid duration "toto"' refute_output rune -1 cscli allowlist add foo 10.10.10.10 10.20.30.40 -d comment -e '1 day' refute_output - assert_stderr 'Error: strconv.Atoi: parsing "1 ": invalid syntax' + assert_stderr 'Error: invalid argument "1 day" for "-e, --expiration" flag: invalid day value in duration "1 day"' - rune -0 cscli allowlist add foo 10.10.10.10 -d comment -e '1d' + rune -0 cscli allowlist add foo 10.10.10.10 -d comment -e '1d12h' assert_output 'added 1 values to allowlist foo' refute_stderr