diff --git a/.golangci.yml b/.golangci.yml index c6dac451f..e9e426cb6 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -184,7 +184,7 @@ linters: maintidx: # raise this after refactoring - under: 15 + under: 18 misspell: locale: US @@ -210,7 +210,7 @@ linters: - name: cognitive-complexity arguments: # lower this after refactoring - - 119 + - 113 - name: comment-spacings disabled: true - name: confusing-results @@ -235,8 +235,8 @@ linters: - name: function-length arguments: # lower this after refactoring - - 111 - - 238 + - 87 + - 198 - name: get-return disabled: true - name: increment-decrement @@ -294,9 +294,7 @@ linters: - -ST1003 - -ST1005 - -ST1012 - - -ST1022 - -QF1003 - - -QF1008 - -QF1012 wsl: diff --git a/cmd/crowdsec-cli/clialert/alerts.go b/cmd/crowdsec-cli/clialert/alerts.go index 88e870ee7..cd9e636ee 100644 --- a/cmd/crowdsec-cli/clialert/alerts.go +++ b/cmd/crowdsec-cli/clialert/alerts.go @@ -104,15 +104,15 @@ func (cli *cliAlerts) alertsToTable(alerts *models.GetAlertsResponse, printMachi if *alerts == nil { // avoid returning "null" in json // could be cleaner if we used slice of alerts directly - fmt.Println("[]") + fmt.Fprintln(os.Stdout, "[]") return nil } x, _ := json.MarshalIndent(alerts, "", " ") - fmt.Print(string(x)) + fmt.Fprint(os.Stdout, string(x)) case "human": if len(*alerts) == 0 { - fmt.Println("No active alerts") + fmt.Fprintln(os.Stdout, "No active alerts") return nil } @@ -156,7 +156,7 @@ func (cli *cliAlerts) displayOneAlert(alert *models.Alert, withDetail bool) erro alertDecisionsTable(color.Output, cfg.Cscli.Color, alert) if len(alert.Meta) > 0 { - fmt.Printf("\n - Context :\n") + fmt.Fprintf(os.Stdout, "\n - Context :\n") sort.Slice(alert.Meta, func(i, j int) bool { return alert.Meta[i].Key < alert.Meta[j].Key }) @@ -183,7 +183,7 @@ func (cli *cliAlerts) displayOneAlert(alert *models.Alert, withDetail bool) erro } if withDetail { - fmt.Printf("\n - Events :\n") + fmt.Fprintf(os.Stdout, "\n - Events :\n") for _, event := range alert.Events { alertEventTable(color.Output, cfg.Cscli.Color, event) @@ -240,7 +240,7 @@ func (cli *cliAlerts) NewCommand() *cobra.Command { func (cli *cliAlerts) list(ctx context.Context, alertListFilter apiclient.AlertsListOpts, limit *int, contained *bool, printMachine bool) error { var err error - *alertListFilter.ScopeEquals, err = SanitizeScope(*alertListFilter.ScopeEquals, *alertListFilter.IPEquals, *alertListFilter.RangeEquals) + alertListFilter.ScopeEquals, err = SanitizeScope(alertListFilter.ScopeEquals, alertListFilter.IPEquals, alertListFilter.RangeEquals) if err != nil { return err } @@ -253,34 +253,6 @@ func (cli *cliAlerts) list(ctx context.Context, alertListFilter apiclient.Alerts *alertListFilter.Limit = 0 } - if *alertListFilter.TypeEquals == "" { - alertListFilter.TypeEquals = nil - } - - if *alertListFilter.ScopeEquals == "" { - alertListFilter.ScopeEquals = nil - } - - if *alertListFilter.ValueEquals == "" { - alertListFilter.ValueEquals = nil - } - - if *alertListFilter.ScenarioEquals == "" { - alertListFilter.ScenarioEquals = nil - } - - if *alertListFilter.IPEquals == "" { - alertListFilter.IPEquals = nil - } - - if *alertListFilter.RangeEquals == "" { - alertListFilter.RangeEquals = nil - } - - if *alertListFilter.OriginEquals == "" { - alertListFilter.OriginEquals = nil - } - if contained != nil && *contained { alertListFilter.Contains = new(bool) } @@ -299,16 +271,16 @@ func (cli *cliAlerts) list(ctx context.Context, alertListFilter apiclient.Alerts func (cli *cliAlerts) newListCmd() *cobra.Command { alertListFilter := apiclient.AlertsListOpts{ - ScopeEquals: new(string), - ValueEquals: new(string), - ScenarioEquals: new(string), - IPEquals: new(string), - RangeEquals: new(string), + ScopeEquals: "", + ValueEquals: "", + ScenarioEquals: "", + IPEquals: "", + RangeEquals: "", Since: cstime.DurationWithDays(0), Until: cstime.DurationWithDays(0), - TypeEquals: new(string), + TypeEquals: "", IncludeCAPI: new(bool), - OriginEquals: new(string), + OriginEquals: "", } limit := new(int) @@ -338,13 +310,13 @@ cscli alerts list --type ban`, flags.BoolVarP(alertListFilter.IncludeCAPI, "all", "a", false, "Include decisions from Central API") flags.Var(&alertListFilter.Until, "until", "restrict to alerts older than until (ie. 4h, 30d)") flags.Var(&alertListFilter.Since, "since", "restrict to alerts newer than since (ie. 4h, 30d)") - flags.StringVarP(alertListFilter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value )") - flags.StringVarP(alertListFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)") - flags.StringVarP(alertListFilter.RangeEquals, "range", "r", "", "restrict to alerts from this range (shorthand for --scope range --value )") - flags.StringVar(alertListFilter.TypeEquals, "type", "", "restrict to alerts with given decision type (ie. ban, captcha)") - flags.StringVar(alertListFilter.ScopeEquals, "scope", "", "restrict to alerts of this scope (ie. ip,range)") - flags.StringVarP(alertListFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope") - flags.StringVar(alertListFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ","))) + flags.StringVarP(&alertListFilter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value )") + flags.StringVarP(&alertListFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)") + flags.StringVarP(&alertListFilter.RangeEquals, "range", "r", "", "restrict to alerts from this range (shorthand for --scope range --value )") + flags.StringVar(&alertListFilter.TypeEquals, "type", "", "restrict to alerts with given decision type (ie. ban, captcha)") + flags.StringVar(&alertListFilter.ScopeEquals, "scope", "", "restrict to alerts of this scope (ie. ip,range)") + flags.StringVarP(&alertListFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope") + flags.StringVar(&alertListFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ","))) flags.BoolVar(contained, "contained", false, "query decisions contained by range") flags.BoolVarP(&printMachine, "machine", "m", false, "print machines that sent alerts") flags.IntVarP(limit, "limit", "l", 50, "limit size of alerts list table (0 to view all alerts)") @@ -356,7 +328,7 @@ func (cli *cliAlerts) delete(ctx context.Context, delFilter apiclient.AlertsDele var err error if !deleteAll { - *delFilter.ScopeEquals, err = SanitizeScope(*delFilter.ScopeEquals, *delFilter.IPEquals, *delFilter.RangeEquals) + delFilter.ScopeEquals, err = SanitizeScope(delFilter.ScopeEquals, delFilter.IPEquals, delFilter.RangeEquals) if err != nil { return err } @@ -365,26 +337,6 @@ func (cli *cliAlerts) delete(ctx context.Context, delFilter apiclient.AlertsDele delFilter.ActiveDecisionEquals = activeDecision } - if *delFilter.ScopeEquals == "" { - delFilter.ScopeEquals = nil - } - - if *delFilter.ValueEquals == "" { - delFilter.ValueEquals = nil - } - - if *delFilter.ScenarioEquals == "" { - delFilter.ScenarioEquals = nil - } - - if *delFilter.IPEquals == "" { - delFilter.IPEquals = nil - } - - if *delFilter.RangeEquals == "" { - delFilter.RangeEquals = nil - } - if contained != nil && *contained { delFilter.Contains = new(bool) } @@ -422,11 +374,11 @@ func (cli *cliAlerts) newDeleteCmd() *cobra.Command { ) delFilter := apiclient.AlertsDeleteOpts{ - ScopeEquals: new(string), - ValueEquals: new(string), - ScenarioEquals: new(string), - IPEquals: new(string), - RangeEquals: new(string), + ScopeEquals: "", + ValueEquals: "", + ScenarioEquals: "", + IPEquals: "", + RangeEquals: "", } contained := new(bool) @@ -445,9 +397,9 @@ cscli alerts delete -s crowdsecurity/ssh-bf"`, if deleteAll { return nil } - if *delFilter.ScopeEquals == "" && *delFilter.ValueEquals == "" && - *delFilter.ScenarioEquals == "" && *delFilter.IPEquals == "" && - *delFilter.RangeEquals == "" && delAlertByID == "" { + if delFilter.ScopeEquals == "" && delFilter.ValueEquals == "" && + delFilter.ScenarioEquals == "" && delFilter.IPEquals == "" && + delFilter.RangeEquals == "" && delAlertByID == "" { _ = cmd.Usage() return errors.New("at least one filter or --all must be specified") } @@ -461,11 +413,11 @@ cscli alerts delete -s crowdsecurity/ssh-bf"`, flags := cmd.Flags() flags.SortFlags = false - flags.StringVar(delFilter.ScopeEquals, "scope", "", "the scope (ie. ip,range)") - flags.StringVarP(delFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope") - flags.StringVarP(delFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)") - flags.StringVarP(delFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value )") - flags.StringVarP(delFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value )") + flags.StringVar(&delFilter.ScopeEquals, "scope", "", "the scope (ie. ip,range)") + flags.StringVarP(&delFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope") + flags.StringVarP(&delFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)") + flags.StringVarP(&delFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value )") + flags.StringVarP(&delFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value )") flags.StringVar(&delAlertByID, "id", "", "alert ID") flags.BoolVarP(&deleteAll, "all", "a", false, "delete all alerts") flags.BoolVar(contained, "contained", false, "query decisions contained by range") @@ -499,14 +451,14 @@ func (cli *cliAlerts) inspect(ctx context.Context, details bool, alertIDs ...str return fmt.Errorf("unable to serialize alert with id %s: %w", alertID, err) } - fmt.Printf("%s\n", string(data)) + fmt.Fprintln(os.Stdout, string(data)) case "raw": data, err := yaml.Marshal(alert) if err != nil { return fmt.Errorf("unable to serialize alert with id %s: %w", alertID, err) } - fmt.Println(string(data)) + fmt.Fprintln(os.Stdout, string(data)) } } @@ -536,7 +488,7 @@ func (cli *cliAlerts) newInspectCmd() *cobra.Command { func (cli *cliAlerts) newFlushCmd() *cobra.Command { var maxItems int - maxAge := cstime.DurationWithDays(7*24*time.Hour) + maxAge := cstime.DurationWithDays(7 * 24 * time.Hour) cmd := &cobra.Command{ Use: `flush`, diff --git a/cmd/crowdsec-cli/cliallowlists/allowlists.go b/cmd/crowdsec-cli/cliallowlists/allowlists.go index 972ee1b47..c28ab5c70 100644 --- a/cmd/crowdsec-cli/cliallowlists/allowlists.go +++ b/cmd/crowdsec-cli/cliallowlists/allowlists.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "net/url" + "os" "slices" "strconv" "strings" @@ -283,7 +284,7 @@ func (cli *cliAllowLists) create(ctx context.Context, db *database.Client, name return err } - fmt.Printf("allowlist '%s' created successfully\n", name) + fmt.Fprintf(os.Stdout, "allowlist '%s' created successfully\n", name) return nil } @@ -392,7 +393,7 @@ func (cli *cliAllowLists) delete(ctx context.Context, db *database.Client, name return err } - fmt.Printf("allowlist '%s' deleted successfully\n", name) + fmt.Fprintf(os.Stdout, "allowlist '%s' deleted successfully\n", name) return nil } @@ -475,7 +476,7 @@ func (cli *cliAllowLists) add(ctx context.Context, db *database.Client, name str } if len(toAdd) == 0 { - fmt.Println("no new values for allowlist") + fmt.Fprintln(os.Stdout, "no new values for allowlist") return nil } @@ -485,7 +486,15 @@ func (cli *cliAllowLists) add(ctx context.Context, db *database.Client, name str } if added > 0 { - fmt.Printf("added %d values to allowlist %s\n", added, name) + fmt.Fprintf(os.Stdout, "added %d values to allowlist %s\n", added, name) + } + + deleted, err := db.ApplyAllowlistsToExistingDecisions(ctx) + if err != nil { + return fmt.Errorf("unable to apply allowlists to existing decisions: %w", err) + } + if deleted > 0 { + fmt.Printf("%d decisions deleted by allowlists\n", deleted) } return nil @@ -614,7 +623,7 @@ func (cli *cliAllowLists) remove(ctx context.Context, db *database.Client, name } if len(toRemove) == 0 { - fmt.Println("no value to remove from allowlist") + fmt.Fprintln(os.Stdout, "no value to remove from allowlist") return nil } @@ -624,7 +633,7 @@ func (cli *cliAllowLists) remove(ctx context.Context, db *database.Client, name } if deleted > 0 { - fmt.Printf("removed %d values from allowlist %s", deleted, name) + fmt.Fprintf(os.Stdout, "removed %d values from allowlist %s", deleted, name) } return nil diff --git a/cmd/crowdsec-cli/clidecision/decisions.go b/cmd/crowdsec-cli/clidecision/decisions.go index c141f1287..c5a67582d 100644 --- a/cmd/crowdsec-cli/clidecision/decisions.go +++ b/cmd/crowdsec-cli/clidecision/decisions.go @@ -103,22 +103,22 @@ func (cli *cliDecisions) decisionsToTable(alerts *models.GetAlertsResponse, prin if *alerts == nil { // avoid returning "null" in `json" // could be cleaner if we used slice of alerts directly - fmt.Println("[]") + fmt.Fprintln(os.Stdout, "[]") return nil } x, _ := json.MarshalIndent(alerts, "", " ") - fmt.Printf("%s", string(x)) + fmt.Fprintln(os.Stdout, string(x)) case "human": if len(*alerts) == 0 { - fmt.Println("No active decisions") + fmt.Fprintln(os.Stdout, "No active decisions") return nil } cli.decisionsTable(color.Output, alerts, printMachine) if skipped > 0 { - fmt.Printf("%d duplicated entries skipped\n", skipped) + fmt.Fprintf(os.Stdout, "%d duplicated entries skipped\n", skipped) } } @@ -175,7 +175,7 @@ func (cli *cliDecisions) NewCommand() *cobra.Command { func (cli *cliDecisions) list(ctx context.Context, filter apiclient.AlertsListOpts, noSimu *bool, contained *bool, printMachine bool) error { var err error - *filter.ScopeEquals, err = clialert.SanitizeScope(*filter.ScopeEquals, *filter.IPEquals, *filter.RangeEquals) + filter.ScopeEquals, err = clialert.SanitizeScope(filter.ScopeEquals, filter.IPEquals, filter.RangeEquals) if err != nil { return err } @@ -193,34 +193,6 @@ func (cli *cliDecisions) list(ctx context.Context, filter apiclient.AlertsListOp *filter.Limit = 0 } - if *filter.TypeEquals == "" { - filter.TypeEquals = nil - } - - if *filter.ValueEquals == "" { - filter.ValueEquals = nil - } - - if *filter.ScopeEquals == "" { - filter.ScopeEquals = nil - } - - if *filter.ScenarioEquals == "" { - filter.ScenarioEquals = nil - } - - if *filter.IPEquals == "" { - filter.IPEquals = nil - } - - if *filter.RangeEquals == "" { - filter.RangeEquals = nil - } - - if *filter.OriginEquals == "" { - filter.OriginEquals = nil - } - if contained != nil && *contained { filter.Contains = new(bool) } @@ -240,15 +212,15 @@ func (cli *cliDecisions) list(ctx context.Context, filter apiclient.AlertsListOp func (cli *cliDecisions) newListCmd() *cobra.Command { filter := apiclient.AlertsListOpts{ - ValueEquals: new(string), - ScopeEquals: new(string), - ScenarioEquals: new(string), - OriginEquals: new(string), - IPEquals: new(string), - RangeEquals: new(string), + ValueEquals: "", + ScopeEquals: "", + ScenarioEquals: "", + OriginEquals: "", + IPEquals: "", + RangeEquals: "", Since: cstime.DurationWithDays(0), Until: cstime.DurationWithDays(0), - TypeEquals: new(string), + TypeEquals: "", IncludeCAPI: new(bool), Limit: new(int), } @@ -278,13 +250,13 @@ cscli decisions list --origin lists --scenario list_name flags.BoolVarP(filter.IncludeCAPI, "all", "a", false, "Include decisions from Central API") flags.Var(&filter.Since, "since", "restrict to alerts newer than since (ie. 4h, 30d)") flags.Var(&filter.Until, "until", "restrict to alerts older than until (ie. 4h, 30d)") - flags.StringVarP(filter.TypeEquals, "type", "t", "", "restrict to this decision type (ie. ban,captcha)") - flags.StringVar(filter.ScopeEquals, "scope", "", "restrict to this scope (ie. ip,range,session)") - flags.StringVar(filter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ","))) - flags.StringVarP(filter.ValueEquals, "value", "v", "", "restrict to this value (ie. 1.2.3.4,userName)") - flags.StringVarP(filter.ScenarioEquals, "scenario", "s", "", "restrict to this scenario (ie. crowdsecurity/ssh-bf)") - flags.StringVarP(filter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value )") - flags.StringVarP(filter.RangeEquals, "range", "r", "", "restrict to alerts from this source range (shorthand for --scope range --value )") + flags.StringVarP(&filter.TypeEquals, "type", "t", "", "restrict to this decision type (ie. ban,captcha)") + flags.StringVar(&filter.ScopeEquals, "scope", "", "restrict to this scope (ie. ip,range,session)") + flags.StringVar(&filter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ","))) + flags.StringVarP(&filter.ValueEquals, "value", "v", "", "restrict to this value (ie. 1.2.3.4,userName)") + flags.StringVarP(&filter.ScenarioEquals, "scenario", "s", "", "restrict to this scenario (ie. crowdsecurity/ssh-bf)") + flags.StringVarP(&filter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value )") + flags.StringVarP(&filter.RangeEquals, "range", "r", "", "restrict to alerts from this source range (shorthand for --scope range --value )") flags.IntVarP(filter.Limit, "limit", "l", 100, "number of alerts to get (use 0 to remove the limit)") flags.BoolVar(NoSimu, "no-simu", false, "exclude decisions in simulation mode") flags.BoolVarP(&printMachine, "machine", "m", false, "print machines that triggered decisions") @@ -428,39 +400,11 @@ func (cli *cliDecisions) delete(ctx context.Context, delFilter apiclient.Decisio var err error /*take care of shorthand options*/ - *delFilter.ScopeEquals, err = clialert.SanitizeScope(*delFilter.ScopeEquals, *delFilter.IPEquals, *delFilter.RangeEquals) + delFilter.ScopeEquals, err = clialert.SanitizeScope(delFilter.ScopeEquals, delFilter.IPEquals, delFilter.RangeEquals) if err != nil { return err } - if *delFilter.ScopeEquals == "" { - delFilter.ScopeEquals = nil - } - - if *delFilter.OriginEquals == "" { - delFilter.OriginEquals = nil - } - - if *delFilter.ValueEquals == "" { - delFilter.ValueEquals = nil - } - - if *delFilter.ScenarioEquals == "" { - delFilter.ScenarioEquals = nil - } - - if *delFilter.TypeEquals == "" { - delFilter.TypeEquals = nil - } - - if *delFilter.IPEquals == "" { - delFilter.IPEquals = nil - } - - if *delFilter.RangeEquals == "" { - delFilter.RangeEquals = nil - } - if contained != nil && *contained { delFilter.Contains = new(bool) } @@ -490,13 +434,13 @@ func (cli *cliDecisions) delete(ctx context.Context, delFilter apiclient.Decisio func (cli *cliDecisions) newDeleteCmd() *cobra.Command { delFilter := apiclient.DecisionsDeleteOpts{ - ScopeEquals: new(string), - ValueEquals: new(string), - TypeEquals: new(string), - IPEquals: new(string), - RangeEquals: new(string), - ScenarioEquals: new(string), - OriginEquals: new(string), + ScopeEquals: "", + ValueEquals: "", + TypeEquals: "", + IPEquals: "", + RangeEquals: "", + ScenarioEquals: "", + OriginEquals: "", } var delDecisionID string @@ -522,10 +466,10 @@ cscli decisions delete --origin lists --scenario list_name if delDecisionAll { return nil } - if *delFilter.ScopeEquals == "" && *delFilter.ValueEquals == "" && - *delFilter.TypeEquals == "" && *delFilter.IPEquals == "" && - *delFilter.RangeEquals == "" && *delFilter.ScenarioEquals == "" && - *delFilter.OriginEquals == "" && delDecisionID == "" { + if delFilter.ScopeEquals == "" && delFilter.ValueEquals == "" && + delFilter.TypeEquals == "" && delFilter.IPEquals == "" && + delFilter.RangeEquals == "" && delFilter.ScenarioEquals == "" && + delFilter.OriginEquals == "" && delDecisionID == "" { _ = cmd.Usage() return errors.New("at least one filter or --all must be specified") } @@ -539,12 +483,12 @@ cscli decisions delete --origin lists --scenario list_name flags := cmd.Flags() flags.SortFlags = false - flags.StringVarP(delFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value )") - flags.StringVarP(delFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value )") - flags.StringVarP(delFilter.TypeEquals, "type", "t", "", "the decision type (ie. ban,captcha)") - flags.StringVarP(delFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope") - flags.StringVarP(delFilter.ScenarioEquals, "scenario", "s", "", "the scenario name (ie. crowdsecurity/ssh-bf)") - flags.StringVar(delFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ","))) + flags.StringVarP(&delFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value )") + flags.StringVarP(&delFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value )") + flags.StringVarP(&delFilter.TypeEquals, "type", "t", "", "the decision type (ie. ban,captcha)") + flags.StringVarP(&delFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope") + flags.StringVarP(&delFilter.ScenarioEquals, "scenario", "s", "", "the scenario name (ie. crowdsecurity/ssh-bf)") + flags.StringVar(&delFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ","))) flags.StringVar(&delDecisionID, "id", "", "decision id") flags.BoolVar(&delDecisionAll, "all", false, "delete all decisions") diff --git a/pkg/apiclient/alerts_service.go b/pkg/apiclient/alerts_service.go index b44049af4..4d30e5976 100644 --- a/pkg/apiclient/alerts_service.go +++ b/pkg/apiclient/alerts_service.go @@ -15,14 +15,14 @@ import ( type AlertsService service type AlertsListOpts struct { - ScopeEquals *string `url:"scope,omitempty"` - ValueEquals *string `url:"value,omitempty"` - ScenarioEquals *string `url:"scenario,omitempty"` - IPEquals *string `url:"ip,omitempty"` - RangeEquals *string `url:"range,omitempty"` - OriginEquals *string `url:"origin,omitempty"` + ScopeEquals string `url:"scope,omitempty"` + ValueEquals string `url:"value,omitempty"` + ScenarioEquals string `url:"scenario,omitempty"` + IPEquals string `url:"ip,omitempty"` + RangeEquals string `url:"range,omitempty"` + OriginEquals string `url:"origin,omitempty"` Since cstime.DurationWithDays `url:"since,omitempty"` - TypeEquals *string `url:"decision_type,omitempty"` + TypeEquals string `url:"decision_type,omitempty"` Until cstime.DurationWithDays `url:"until,omitempty"` IncludeSimulated *bool `url:"simulated,omitempty"` ActiveDecisionEquals *bool `url:"has_active_decision,omitempty"` @@ -33,16 +33,16 @@ type AlertsListOpts struct { } type AlertsDeleteOpts struct { - ScopeEquals *string `url:"scope,omitempty"` - ValueEquals *string `url:"value,omitempty"` - ScenarioEquals *string `url:"scenario,omitempty"` - IPEquals *string `url:"ip,omitempty"` - RangeEquals *string `url:"range,omitempty"` + ScopeEquals string `url:"scope,omitempty"` + ValueEquals string `url:"value,omitempty"` + ScenarioEquals string `url:"scenario,omitempty"` + IPEquals string `url:"ip,omitempty"` + RangeEquals string `url:"range,omitempty"` Since cstime.DurationWithDays `url:"since,omitempty"` Until cstime.DurationWithDays `url:"until,omitempty"` - OriginEquals *string `url:"origin,omitempty"` + OriginEquals string `url:"origin,omitempty"` ActiveDecisionEquals *bool `url:"has_active_decision,omitempty"` - SourceEquals *string `url:"alert_source,omitempty"` + SourceEquals string `url:"alert_source,omitempty"` Contains *bool `url:"contains,omitempty"` Limit *int `url:"limit,omitempty"` ListOpts diff --git a/pkg/apiclient/alerts_service_test.go b/pkg/apiclient/alerts_service_test.go index 24b66937f..eed1598c8 100644 --- a/pkg/apiclient/alerts_service_test.go +++ b/pkg/apiclient/alerts_service_test.go @@ -18,6 +18,7 @@ import ( func TestAlertsListAsMachine(t *testing.T) { ctx := t.Context() + log.SetLevel(log.DebugLevel) mux, urlx, teardown := setup() @@ -189,7 +190,7 @@ func TestAlertsListAsMachine(t *testing.T) { assert.Equal(t, expected, *alerts) // this one doesn't - filter := AlertsListOpts{IPEquals: ptr.Of("1.2.3.4")} + filter := AlertsListOpts{IPEquals: "1.2.3.4"} alerts, resp, err = client.Alerts.List(ctx, filter) require.NoError(t, err) @@ -199,6 +200,7 @@ func TestAlertsListAsMachine(t *testing.T) { func TestAlertsGetAsMachine(t *testing.T) { ctx := t.Context() + log.SetLevel(log.DebugLevel) mux, urlx, teardown := setup() @@ -367,6 +369,7 @@ func TestAlertsGetAsMachine(t *testing.T) { func TestAlertsCreateAsMachine(t *testing.T) { ctx := t.Context() + log.SetLevel(log.DebugLevel) mux, urlx, teardown := setup() @@ -410,6 +413,7 @@ func TestAlertsCreateAsMachine(t *testing.T) { func TestAlertsDeleteAsMachine(t *testing.T) { ctx := t.Context() + log.SetLevel(log.DebugLevel) mux, urlx, teardown := setup() @@ -442,7 +446,7 @@ func TestAlertsDeleteAsMachine(t *testing.T) { defer teardown() - alert := AlertsDeleteOpts{IPEquals: ptr.Of("1.2.3.4")} + alert := AlertsDeleteOpts{IPEquals: "1.2.3.4"} alerts, resp, err := client.Alerts.Delete(ctx, alert) require.NoError(t, err) diff --git a/pkg/apiclient/auth_key_test.go b/pkg/apiclient/auth_key_test.go index aa92e03bb..1e5f95acc 100644 --- a/pkg/apiclient/auth_key_test.go +++ b/pkg/apiclient/auth_key_test.go @@ -10,11 +10,11 @@ import ( "github.com/stretchr/testify/require" "github.com/crowdsecurity/go-cs-lib/cstest" - "github.com/crowdsecurity/go-cs-lib/ptr" ) func TestApiAuth(t *testing.T) { ctx := t.Context() + log.SetLevel(log.TraceLevel) mux, urlx, teardown := setup() @@ -40,7 +40,7 @@ func TestApiAuth(t *testing.T) { defer teardown() - //ok no answer + // ok no answer auth := &APIKeyTransport{ APIKey: "ixu", } @@ -48,12 +48,12 @@ func TestApiAuth(t *testing.T) { newcli, err := NewDefaultClient(apiURL, "v1", "toto", auth.Client()) require.NoError(t, err) - alert := DecisionsListOpts{IPEquals: ptr.Of("1.2.3.4")} + alert := DecisionsListOpts{IPEquals: "1.2.3.4"} _, resp, err := newcli.Decisions.List(ctx, alert) require.NoError(t, err) assert.Equal(t, http.StatusOK, resp.Response.StatusCode) - //ko bad token + // ko bad token auth = &APIKeyTransport{ APIKey: "bad", } @@ -69,7 +69,7 @@ func TestApiAuth(t *testing.T) { cstest.RequireErrorMessage(t, err, "API error: access forbidden") - //ko empty token + // ko empty token auth = &APIKeyTransport{} newcli, err = NewDefaultClient(apiURL, "v1", "toto", auth.Client()) diff --git a/pkg/apiclient/decisions_service.go b/pkg/apiclient/decisions_service.go index a1810e831..c222e2ddb 100644 --- a/pkg/apiclient/decisions_service.go +++ b/pkg/apiclient/decisions_service.go @@ -20,12 +20,12 @@ import ( type DecisionsService service type DecisionsListOpts struct { - ScopeEquals *string `url:"scope,omitempty"` - ValueEquals *string `url:"value,omitempty"` - TypeEquals *string `url:"type,omitempty"` - IPEquals *string `url:"ip,omitempty"` - RangeEquals *string `url:"range,omitempty"` - Contains *bool `url:"contains,omitempty"` + ScopeEquals string `url:"scope,omitempty"` + ValueEquals string `url:"value,omitempty"` + TypeEquals string `url:"type,omitempty"` + IPEquals string `url:"ip,omitempty"` + RangeEquals string `url:"range,omitempty"` + Contains *bool `url:"contains,omitempty"` ListOpts } @@ -60,15 +60,15 @@ func (o *DecisionsStreamOpts) addQueryParamsToURL(url string) (string, error) { } type DecisionsDeleteOpts struct { - ScopeEquals *string `url:"scope,omitempty"` - ValueEquals *string `url:"value,omitempty"` - TypeEquals *string `url:"type,omitempty"` - IPEquals *string `url:"ip,omitempty"` - RangeEquals *string `url:"range,omitempty"` - Contains *bool `url:"contains,omitempty"` - OriginEquals *string `url:"origin,omitempty"` + ScopeEquals string `url:"scope,omitempty"` + ValueEquals string `url:"value,omitempty"` + TypeEquals string `url:"type,omitempty"` + IPEquals string `url:"ip,omitempty"` + RangeEquals string `url:"range,omitempty"` + Contains *bool `url:"contains,omitempty"` + OriginEquals string `url:"origin,omitempty"` // - ScenarioEquals *string `url:"scenario,omitempty"` + ScenarioEquals string `url:"scenario,omitempty"` ListOpts } diff --git a/pkg/apiclient/decisions_service_test.go b/pkg/apiclient/decisions_service_test.go index c16abed64..c9e555e92 100644 --- a/pkg/apiclient/decisions_service_test.go +++ b/pkg/apiclient/decisions_service_test.go @@ -19,6 +19,7 @@ import ( func TestDecisionsList(t *testing.T) { ctx := t.Context() + log.SetLevel(log.DebugLevel) mux, urlx, teardown := setup() @@ -64,15 +65,13 @@ func TestDecisionsList(t *testing.T) { } // OK decisions - decisionsFilter := DecisionsListOpts{IPEquals: ptr.Of("1.2.3.4")} - decisions, resp, err := newcli.Decisions.List(ctx, decisionsFilter) + decisions, resp, err := newcli.Decisions.List(ctx, DecisionsListOpts{IPEquals: "1.2.3.4"}) require.NoError(t, err) assert.Equal(t, http.StatusOK, resp.Response.StatusCode) assert.Equal(t, *expected, *decisions) // Empty return - decisionsFilter = DecisionsListOpts{IPEquals: ptr.Of("1.2.3.5")} - decisions, resp, err = newcli.Decisions.List(ctx, decisionsFilter) + decisions, resp, err = newcli.Decisions.List(ctx, DecisionsListOpts{IPEquals: "1.2.3.5"}) require.NoError(t, err) assert.Equal(t, http.StatusOK, resp.Response.StatusCode) assert.Empty(t, *decisions) @@ -80,6 +79,7 @@ func TestDecisionsList(t *testing.T) { func TestDecisionsStream(t *testing.T) { ctx := t.Context() + log.SetLevel(log.DebugLevel) mux, urlx, teardown := setup() @@ -156,6 +156,7 @@ func TestDecisionsStream(t *testing.T) { func TestDecisionsStreamV3Compatibility(t *testing.T) { ctx := t.Context() + log.SetLevel(log.DebugLevel) mux, urlx, teardown := setupWithPrefix("v3") @@ -224,6 +225,7 @@ func TestDecisionsStreamV3Compatibility(t *testing.T) { func TestDecisionsStreamV3(t *testing.T) { ctx := t.Context() + log.SetLevel(log.DebugLevel) mux, urlx, teardown := setupWithPrefix("v3") @@ -297,6 +299,7 @@ func TestDecisionsStreamV3(t *testing.T) { func TestDecisionsFromBlocklist(t *testing.T) { ctx := t.Context() + log.SetLevel(log.DebugLevel) mux, urlx, teardown := setupWithPrefix("v3") @@ -429,10 +432,7 @@ func TestDeleteDecisions(t *testing.T) { }) require.NoError(t, err) - filters := DecisionsDeleteOpts{IPEquals: new(string)} - *filters.IPEquals = "1.2.3.4" - - deleted, _, err := client.Decisions.Delete(ctx, filters) + deleted, _, err := client.Decisions.Delete(ctx, DecisionsDeleteOpts{IPEquals: "1.2.3.4"}) require.NoError(t, err) assert.Equal(t, "1", deleted.NbDeleted) } diff --git a/pkg/apiserver/apic.go b/pkg/apiserver/apic.go index 2a9176dd9..669ac5ce3 100644 --- a/pkg/apiserver/apic.go +++ b/pkg/apiserver/apic.go @@ -661,6 +661,8 @@ func fillAlertsWithDecisions(alerts []*models.Alert, decisions []*models.Decisio func (a *apic) PullTop(ctx context.Context, forcePull bool) error { var err error + hasPulledAllowlists := false + // A mutex with TryLock would be a bit simpler // But go does not guarantee that TryLock will be able to acquire the lock even if it is available select { @@ -722,7 +724,7 @@ func (a *apic) PullTop(ctx context.Context, forcePull bool) error { // process deleted decisions nbDeleted, err := a.HandleDeletedDecisionsV3(ctx, data.Deleted, deleteCounters) if err != nil { - return err + log.Errorf("could not delete decisions from CAPI: %s", err) } log.Printf("capi/community-blocklist : %d explicit deletions", nbDeleted) @@ -730,8 +732,9 @@ func (a *apic) PullTop(ctx context.Context, forcePull bool) error { // Update allowlists before processing decisions if data.Links != nil { if len(data.Links.Allowlists) > 0 { + hasPulledAllowlists = true if err := a.UpdateAllowlists(ctx, data.Links.Allowlists, forcePull); err != nil { - return fmt.Errorf("while updating allowlists: %w", err) + log.Errorf("could not update allowlists from CAPI: %s", err) } } } @@ -748,7 +751,7 @@ func (a *apic) PullTop(ctx context.Context, forcePull bool) error { err = a.SaveAlerts(ctx, alertsFromCapi, addCounters, deleteCounters) if err != nil { - return fmt.Errorf("while saving alerts: %w", err) + log.Errorf("could not save alert for CAPI pull: %s", err) } } else { if a.pullCommunity { @@ -762,11 +765,21 @@ func (a *apic) PullTop(ctx context.Context, forcePull bool) error { if data.Links != nil { if len(data.Links.Blocklists) > 0 { if err := a.UpdateBlocklists(ctx, data.Links.Blocklists, addCounters, forcePull); err != nil { - return fmt.Errorf("while updating blocklists: %w", err) + log.Errorf("could not update blocklists from CAPI: %s", err) } } } + if hasPulledAllowlists { + deleted, err := a.dbClient.ApplyAllowlistsToExistingDecisions(ctx) + if err != nil { + log.Errorf("could not apply allowlists to existing decisions: %s", err) + } + if deleted > 0 { + log.Infof("deleted %d decisions from allowlists", deleted) + } + } + return nil } diff --git a/pkg/apiserver/papi_cmd.go b/pkg/apiserver/papi_cmd.go index 2e48ef4ea..f9ecbf359 100644 --- a/pkg/apiserver/papi_cmd.go +++ b/pkg/apiserver/papi_cmd.go @@ -264,6 +264,14 @@ func ManagementCmd(message *Message, p *Papi, sync bool) error { if err != nil { return fmt.Errorf("failed to force pull operation: %w", err) } + + deleted, err := p.DBClient.ApplyAllowlistsToExistingDecisions(ctx) + if err != nil { + log.Errorf("could not apply allowlists to existing decisions: %s", err) + } + if deleted > 0 { + log.Infof("deleted %d decisions from allowlists", deleted) + } } case "allowlist_unsubscribe": data, err := json.Marshal(message.Data) diff --git a/pkg/database/allowlists.go b/pkg/database/allowlists.go index d14958f61..a8815a7f6 100644 --- a/pkg/database/allowlists.go +++ b/pkg/database/allowlists.go @@ -12,6 +12,8 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/allowlist" "github.com/crowdsecurity/crowdsec/pkg/database/ent/allowlistitem" + "github.com/crowdsecurity/crowdsec/pkg/database/ent/decision" + "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -389,3 +391,96 @@ func (c *Client) GetAllowlistsContentForAPIC(ctx context.Context) ([]net.IP, []* return ips, nets, nil } + +func (c *Client) ApplyAllowlistsToExistingDecisions(ctx context.Context) (int, error) { + // Soft delete (set expiration to now) all decisions that matches any allowlist + + totalCount := 0 + + // Get all non-expired allowlist items + // We will match them one by one against all decisions + allowlistItems, err := c.Ent.AllowListItem.Query(). + Where( + allowlistitem.Or( + allowlistitem.ExpiresAtGTE(time.Now().UTC()), + allowlistitem.ExpiresAtIsNil(), + ), + ).All(ctx) + if err != nil { + return 0, fmt.Errorf("unable to get allowlist items: %w", err) + } + + now := time.Now().UTC() + + for _, item := range allowlistItems { + updateQuery := c.Ent.Decision.Update().SetUntil(now).Where(decision.UntilGTE(now)) + switch item.IPSize { + case 4: + updateQuery = updateQuery.Where( + decision.And( + decision.IPSizeEQ(4), + decision.Or( + decision.And( + decision.StartIPLTE(item.StartIP), + decision.EndIPGTE(item.EndIP), + ), + decision.And( + decision.StartIPGTE(item.StartIP), + decision.EndIPLTE(item.EndIP), + ), + ))) + case 16: + updateQuery = updateQuery.Where( + decision.And( + decision.IPSizeEQ(16), + decision.Or( + decision.And( + decision.Or( + decision.StartIPLT(item.StartIP), + decision.And( + decision.StartIPEQ(item.StartIP), + decision.StartSuffixLTE(item.StartSuffix), + )), + decision.Or( + decision.EndIPGT(item.EndIP), + decision.And( + decision.EndIPEQ(item.EndIP), + decision.EndSuffixGTE(item.EndSuffix), + ), + ), + ), + decision.And( + decision.Or( + decision.StartIPGT(item.StartIP), + decision.And( + decision.StartIPEQ(item.StartIP), + decision.StartSuffixGTE(item.StartSuffix), + )), + decision.Or( + decision.EndIPLT(item.EndIP), + decision.And( + decision.EndIPEQ(item.EndIP), + decision.EndSuffixLTE(item.EndSuffix), + ), + ), + ), + ), + ), + ) + default: + // This should never happen + // But better safe than sorry and just skip it instead of expiring all decisions + c.Log.Errorf("unexpected IP size %d for allowlist item %s", item.IPSize, item.Value) + continue + } + // Update the decisions + count, err := updateQuery.Save(ctx) + if err != nil { + c.Log.Errorf("unable to expire existing decisions: %s", err) + continue + } + totalCount += count + } + + return totalCount, nil +} diff --git a/test/bats/cscli-allowlists.bats b/test/bats/cscli-allowlists.bats index e146e8750..d357f29f9 100644 --- a/test/bats/cscli-allowlists.bats +++ b/test/bats/cscli-allowlists.bats @@ -246,3 +246,58 @@ teardown() { rune -0 jq 'del(.created_at) | del(.updated_at) | del(.items.[].created_at) | del(.items.[].expiration)' <(output) assert_json '{"description":"a foo","items":[],"name":"foo"}' } + +@test "allowlists expire active decisions" { + rune -0 cscli decisions add -i 1.2.3.4 + rune -0 cscli decisions add -r 2.3.4.0/24 + rune -0 cscli decisions add -i 5.4.3.42 + rune -0 cscli decisions add -r 6.5.4.0/24 + rune -0 cscli decisions add -r 10.0.0.0/23 + + rune -0 cscli decisions list -o json + rune -0 jq -r 'sort_by(.decisions[].value) | .[].decisions[0].value' <(output) + assert_output - <<-EOT + 1.2.3.4 + 10.0.0.0/23 + 2.3.4.0/24 + 5.4.3.42 + 6.5.4.0/24 + EOT + + rune -0 cscli allowlists create foo -d "foo" + + # add an allowlist that matches exactly + rune -0 cscli allowlists add foo 1.2.3.4 + if is_db_mysql; then sleep 2; fi + # it should not be here anymore + rune -0 cscli decisions list -o json + rune -0 jq -e 'any(.[].decisions[]; .value == "1.2.3.4") | not' <(output) + + # allowlist an IP belonging to a range + rune -0 cscli allowlist add foo 2.3.4.42 + if is_db_mysql; then sleep 2; fi + rune -0 cscli decisions list -o json + rune -0 jq -e 'any(.[].decisions[]; .value == "2.3.4.0/24") | not' <(output) + + # allowlist a range with an active decision inside + rune -0 cscli allowlist add foo 5.4.3.0/24 + if is_db_mysql; then sleep 2; fi + rune -0 cscli decisions list -o json + rune -0 jq -e 'any(.[].decisions[]; .value == "5.4.3.42") | not' <(output) + + # allowlist a range inside a range for which we have a decision + rune -0 cscli allowlist add foo 6.5.4.0/25 + if is_db_mysql; then sleep 2; fi + rune -0 cscli decisions list -o json + rune -0 jq -e 'any(.[].decisions[]; .value == "6.5.4.0/24") | not' <(output) + + # allowlist a range bigger than a range for which we have a decision + rune -0 cscli allowlist add foo 10.0.0.0/24 + if is_db_mysql; then sleep 2; fi + rune -0 cscli decisions list -o json + rune -0 jq -e 'any(.[].decisions[]; .value == "10.0.0.0/24") | not' <(output) + + # sanity check no more active decisions + rune -0 cscli decisions list -o json + assert_json [] +} diff --git a/test/lib/config/config-global b/test/lib/config/config-global index 83d95e68e..014b7cc19 100755 --- a/test/lib/config/config-global +++ b/test/lib/config/config-global @@ -116,7 +116,7 @@ load_init_data() { dump_backend="$(cat "${LOCAL_INIT_DIR}/.backend")" if [[ "${DB_BACKEND}" != "${dump_backend}" ]]; then - die "Can't run with backend '${DB_BACKEND}' because the test data was built with '${dump_backend}'" + die "Can't run with backend '${DB_BACKEND}' because 'make bats-fixture' was ran with '${dump_backend}'" fi remove_init_data diff --git a/test/lib/config/config-local b/test/lib/config/config-local index 54ac8550c..e6588c61c 100755 --- a/test/lib/config/config-local +++ b/test/lib/config/config-local @@ -168,7 +168,7 @@ load_init_data() { dump_backend="$(cat "${LOCAL_INIT_DIR}/.backend")" if [[ "${DB_BACKEND}" != "${dump_backend}" ]]; then - die "Can't run with backend '${DB_BACKEND}' because the test data was built with '${dump_backend}'" + die "Can't run with backend '${DB_BACKEND}' because 'make bats-fixture' was ran with '${dump_backend}'" fi remove_init_data diff --git a/test/run-tests b/test/run-tests index e7609188c..ed9d6f553 100755 --- a/test/run-tests +++ b/test/run-tests @@ -26,7 +26,7 @@ fi dump_backend="$(cat "$LOCAL_INIT_DIR/.backend")" if [[ "$DB_BACKEND" != "$dump_backend" ]]; then - die "Can't run with backend '$DB_BACKEND' because the test data was build with '$dump_backend'" + die "Can't run with backend '$DB_BACKEND' because 'make bats-fixture' was ran with '$dump_backend'" fi if [[ $# -ge 1 ]]; then