mirrors/crowdsec
1
0
Fork 0
mirror of https://github.com/crowdsecurity/crowdsec.git synced 2025-05-12 04:45:52 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix #1552 (#1569)

Browse source
This commit is contained in:
AlteredCoder 2022-06-22 10:29:02 +02:00 • committed by GitHub
parent a1d5a02646
commit 0a39066f9d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 1680 additions and 240 deletions
Download patch file Download diff file Expand all files Collapse all files

3
cmd/crowdsec-cli/decisions.go
View file

@ -284,7 +284,7 @@ cscli decisions list -t ban
cmdDecisionsList.Flags().StringVar(filter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)") cmdDecisionsList.Flags().StringVar(filter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
cmdDecisionsList.Flags().StringVarP(filter.TypeEquals, "type", "t", "", "restrict to this decision type (ie. ban,captcha)") cmdDecisionsList.Flags().StringVarP(filter.TypeEquals, "type", "t", "", "restrict to this decision type (ie. ban,captcha)")
cmdDecisionsList.Flags().StringVar(filter.ScopeEquals, "scope", "", "restrict to this scope (ie. ip,range,session)") cmdDecisionsList.Flags().StringVar(filter.ScopeEquals, "scope", "", "restrict to this scope (ie. ip,range,session)")
cmdDecisionsList.Flags().StringVar(filter.OriginEquals, "origin", "", "restrict to this origin (ie. lists,CAPI,cscli)") cmdDecisionsList.Flags().StringVar(filter.OriginEquals, "origin", "", "restrict to this origin (ie. lists,CAPI,cscli,cscli-import,crowdsec)")
cmdDecisionsList.Flags().StringVarP(filter.ValueEquals, "value", "v", "", "restrict to this value (ie. 1.2.3.4,userName)") cmdDecisionsList.Flags().StringVarP(filter.ValueEquals, "value", "v", "", "restrict to this value (ie. 1.2.3.4,userName)")
cmdDecisionsList.Flags().StringVarP(filter.ScenarioEquals, "scenario", "s", "", "restrict to this scenario (ie. crowdsecurity/ssh-bf)") cmdDecisionsList.Flags().StringVarP(filter.ScenarioEquals, "scenario", "s", "", "restrict to this scenario (ie. crowdsecurity/ssh-bf)")
cmdDecisionsList.Flags().StringVarP(filter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)") cmdDecisionsList.Flags().StringVarP(filter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
@ -419,7 +419,6 @@ cscli decisions add --scope username --value foobar
Aliases: []string{"remove"}, Aliases: []string{"remove"},
Example: `cscli decisions delete -r 1.2.3.0/24 Example: `cscli decisions delete -r 1.2.3.0/24
cscli decisions delete -i 1.2.3.4 cscli decisions delete -i 1.2.3.4
cscli decisions delete -s crowdsecurity/ssh-bf
cscli decisions delete --id 42 cscli decisions delete --id 42
cscli decisions delete --type captcha cscli decisions delete --type captcha
`, `,

3
pkg/apiserver/alerts_test.go
View file

@ -23,6 +23,7 @@ type LAPI struct {
loginResp models.WatcherAuthResponse loginResp models.WatcherAuthResponse
bouncerKey string bouncerKey string
t *testing.T t *testing.T
DBConfig *csconfig.DatabaseCfg
} }
func SetupLAPITest(t *testing.T) LAPI { func SetupLAPITest(t *testing.T) LAPI {
@ -36,10 +37,12 @@ func SetupLAPITest(t *testing.T) LAPI {
if err != nil { if err != nil {
t.Fatalf("%s", err.Error()) t.Fatalf("%s", err.Error())
} }
return LAPI{ return LAPI{
router: router, router: router,
loginResp: loginResp, loginResp: loginResp,
bouncerKey: APIKey, bouncerKey: APIKey,
DBConfig: config.API.Server.DbConfig,
} }
} }

29
pkg/apiserver/controllers/v1/decisions.go
View file

@ -11,9 +11,20 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
) )
func FormatDecisions(decisions []*ent.Decision) ([]*models.Decision, error) { //Format decisions for the bouncers, and deduplicate them by keeping only the longest one
func FormatDecisions(decisions []*ent.Decision, dedup bool) ([]*models.Decision, error) {
var results []*models.Decision var results []*models.Decision
seen := make(map[string]struct{}, 0)
for _, dbDecision := range decisions { for _, dbDecision := range decisions {
if dedup {
key := dbDecision.Value + dbDecision.Scope + dbDecision.Type
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
}
duration := dbDecision.Until.Sub(time.Now().UTC()).String() duration := dbDecision.Until.Sub(time.Now().UTC()).String()
decision := models.Decision{ decision := models.Decision{
ID: int64(dbDecision.ID), ID: int64(dbDecision.ID),
@ -46,7 +57,7 @@ func (c *Controller) GetDecision(gctx *gin.Context) {
return return
} }
results, err = FormatDecisions(data) results, err = FormatDecisions(data, false)
if err != nil { if err != nil {
gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()}) gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()})
return return
@ -82,14 +93,14 @@ func (c *Controller) DeleteDecisionById(gctx *gin.Context) {
gctx.JSON(http.StatusBadRequest, gin.H{"message": "decision_id must be valid integer"}) gctx.JSON(http.StatusBadRequest, gin.H{"message": "decision_id must be valid integer"})
return return
} }
err = c.DBClient.SoftDeleteDecisionByID(decisionID) nbDeleted, err := c.DBClient.SoftDeleteDecisionByID(decisionID)
if err != nil { if err != nil {
c.HandleDBErrors(gctx, err) c.HandleDBErrors(gctx, err)
return return
} }
deleteDecisionResp := models.DeleteDecisionResponse{ deleteDecisionResp := models.DeleteDecisionResponse{
NbDeleted: "1", NbDeleted: strconv.Itoa(nbDeleted),
} }
gctx.JSON(http.StatusOK, deleteDecisionResp) gctx.JSON(http.StatusOK, deleteDecisionResp)
@ -138,7 +149,8 @@ func (c *Controller) StreamDecision(gctx *gin.Context) {
gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()}) gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()})
return return
} }
ret["new"], err = FormatDecisions(data) //data = KeepLongestDecision(data)
ret["new"], err = FormatDecisions(data, true)
if err != nil { if err != nil {
log.Errorf("unable to format expired decision for '%s' : %v", bouncerInfo.Name, err) log.Errorf("unable to format expired decision for '%s' : %v", bouncerInfo.Name, err)
gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()}) gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()})
@ -152,7 +164,7 @@ func (c *Controller) StreamDecision(gctx *gin.Context) {
gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()}) gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()})
return return
} }
ret["deleted"], err = FormatDecisions(data) ret["deleted"], err = FormatDecisions(data, true)
if err != nil { if err != nil {
log.Errorf("unable to format expired decision for '%s' : %v", bouncerInfo.Name, err) log.Errorf("unable to format expired decision for '%s' : %v", bouncerInfo.Name, err)
gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()}) gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()})
@ -180,7 +192,8 @@ func (c *Controller) StreamDecision(gctx *gin.Context) {
gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()}) gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()})
return return
} }
ret["new"], err = FormatDecisions(data) //data = KeepLongestDecision(data)
ret["new"], err = FormatDecisions(data, true)
if err != nil { if err != nil {
log.Errorf("unable to format new decision for '%s' : %v", bouncerInfo.Name, err) log.Errorf("unable to format new decision for '%s' : %v", bouncerInfo.Name, err)
gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()}) gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()})
@ -194,7 +207,7 @@ func (c *Controller) StreamDecision(gctx *gin.Context) {
gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()}) gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()})
return return
} }
ret["deleted"], err = FormatDecisions(data) ret["deleted"], err = FormatDecisions(data, true)
if err != nil { if err != nil {
log.Errorf("unable to format expired decision for '%s' : %v", bouncerInfo.Name, err) log.Errorf("unable to format expired decision for '%s' : %v", bouncerInfo.Name, err)
gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()}) gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()})

1226
pkg/apiserver/decisions_test.go
View file

File diff suppressed because it is too large Load diff

266
pkg/apiserver/tests/alert_duplicate.json Normal file
View file

@ -0,0 +1,266 @@
[
{
"id": 42,
"machine_id": "test",
"capacity": 1,
"created_at": "2020-10-09T10:00:10Z",
"decisions": [
{
"duration": "1h",
"origin": "test",
"scenario": "crowdsecurity/test",
"scope": "Ip",
"value": "127.0.0.1",
"type": "ban"
}
],
"source": {
"ip": "127.0.0.1",
"range": "127.0.0.1/32",
"scope": "ip",
"value": "127.0.0.1"
},
"Events": [
],
"events_count": 1,
"leakspeed": "0.5s",
"message": "test",
"scenario_hash": "hashtest",
"scenario_version": "v1",
"simulated": false,
"scenario": "crowdsecurity/test",
"start_at": "2020-10-09T10:00:01Z",
"stop_at": "2020-10-09T10:00:05Z"
},
{
"id": 44,
"machine_id": "test",
"created_at": "2020-10-09T10:00:10Z",
"decisions": [
{
"duration": "3h",
"origin": "another_origin",
"scenario": "crowdsecurity/ssh_bf",
"scope": "Ip",
"value": "127.0.0.1",
"type": "ban"
}
],
"source": {
"ip": "127.0.0.1",
"range": "127.0.0.1/32",
"scope": "ip",
"value": "127.0.0.1"
},
"Events": [
],
"events_count": 1,
"leakspeed": "0.5s",
"message": "test",
"scenario_hash": "hashtest",
"scenario_version": "v1",
"simulated": false,
"capacity": 1,
"scenario": "crowdsecurity/ssh_bf",
"start_at": "2020-10-09T10:00:01Z",
"stop_at": "2020-10-09T10:00:05Z"
},
{
"id": 45,
"machine_id": "test",
"created_at": "2020-10-09T10:00:10Z",
"decisions": [
{
"duration": "5h",
"origin": "test",
"scenario": "crowdsecurity/longest",
"scope": "Ip",
"value": "127.0.0.1",
"type": "ban"
}
],
"source": {
"ip": "127.0.0.1",
"range": "127.0.0.1/32",
"scope": "ip",
"value": "127.0.0.1"
},
"Events": [
],
"events_count": 1,
"leakspeed": "0.5s",
"message": "test",
"scenario_hash": "hashtest",
"scenario_version": "v1",
"simulated": false,
"capacity": 1,
"scenario": "crowdsecurity/longest",
"start_at": "2020-10-09T10:00:01Z",
"stop_at": "2020-10-09T10:00:05Z"
},
{
"id": 46,
"machine_id": "test",
"created_at": "2020-10-09T10:00:10Z",
"decisions": [
{
"duration": "3h",
"origin": "test",
"scenario": "crowdsecurity/test",
"scope": "Ip",
"value": "127.0.0.2",
"type": "ban"
}
],
"source": {
"ip": "127.0.0.2",
"range": "127.0.0.2/32",
"scope": "ip",
"value": "127.0.0.2"
},
"Events": [
],
"events_count": 1,
"leakspeed": "0.5s",
"message": "test",
"scenario_hash": "hashtest",
"scenario_version": "v1",
"simulated": false,
"capacity": 1,
"scenario": "crowdsecurity/test",
"start_at": "2020-10-09T10:00:01Z",
"stop_at": "2020-10-09T10:00:05Z"
},
{
"id": 47,
"machine_id": "test",
"created_at": "2020-10-09T10:00:10Z",
"decisions": [
{
"duration": "3h",
"origin": "test",
"scenario": "crowdsecurity/ssh_bf",
"scope": "Ip",
"value": "127.0.0.2",
"type": "ban"
}
],
"source": {
"ip": "127.0.0.2",
"range": "127.0.0.2/32",
"scope": "ip",
"value": "127.0.0.2"
},
"Events": [
],
"events_count": 1,
"leakspeed": "0.5s",
"message": "test",
"scenario_hash": "hashtest",
"scenario_version": "v1",
"simulated": false,
"capacity": 1,
"scenario": "crowdsecurity/ssh_bf",
"start_at": "2020-10-09T10:00:01Z",
"stop_at": "2020-10-09T10:00:05Z"
},
{
"id": 48,
"machine_id": "test",
"created_at": "2020-10-09T10:00:10Z",
"decisions": [
{
"duration": "1h",
"origin": "test",
"scenario": "crowdsecurity/ssh_bf",
"scope": "Ip",
"value": "127.0.0.2",
"type": "ban"
}
],
"source": {
"ip": "127.0.0.2",
"range": "127.0.0.2/32",
"scope": "ip",
"value": "127.0.0.2"
},
"Events": [
],
"events_count": 1,
"leakspeed": "0.5s",
"message": "test",
"scenario_hash": "hashtest",
"scenario_version": "v1",
"simulated": false,
"capacity": 1,
"scenario": "crowdsecurity/ssh_bf",
"start_at": "2020-10-09T10:00:01Z",
"stop_at": "2020-10-09T10:00:05Z"
},
{
"id": 49,
"machine_id": "test",
"created_at": "2020-10-09T10:00:10Z",
"decisions": [
{
"duration": "2h",
"origin": "another_origin",
"scenario": "crowdsecurity/test",
"scope": "Ip",
"value": "127.0.0.2",
"type": "ban"
}
],
"source": {
"ip": "127.0.0.2",
"range": "127.0.0.2/32",
"scope": "ip",
"value": "127.0.0.2"
},
"Events": [
],
"events_count": 1,
"leakspeed": "0.5s",
"message": "test",
"scenario_hash": "hashtest",
"scenario_version": "v1",
"simulated": false,
"capacity": 1,
"scenario": "crowdsecurity/test",
"start_at": "2020-10-09T10:00:01Z",
"stop_at": "2020-10-09T10:00:05Z"
},
{
"id": 50,
"machine_id": "test",
"created_at": "2020-10-09T10:00:10Z",
"decisions": [
{
"duration": "3h",
"origin": "test",
"scenario": "crowdsecurity/test",
"scope": "Ip",
"value": "127.0.0.2",
"type": "captcha"
}
],
"source": {
"ip": "127.0.0.2",
"range": "127.0.0.2/32",
"scope": "ip",
"value": "127.0.0.2"
},
"Events": [
],
"events_count": 1,
"leakspeed": "0.5s",
"message": "test",
"scenario_hash": "hashtest",
"scenario_version": "v1",
"simulated": false,
"capacity": 1,
"scenario": "crowdsecurity/test",
"start_at": "2020-10-09T10:00:01Z",
"stop_at": "2020-10-09T10:00:05Z"
}
]

148
pkg/database/decisions.go
View file

@ -15,16 +15,19 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
) )
func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string][]string) (*ent.DecisionQuery, error) { func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string][]string) (*ent.DecisionQuery, []*sql.Predicate, error) {
var err error var err error
var start_ip, start_sfx, end_ip, end_sfx int64 var start_ip, start_sfx, end_ip, end_sfx int64
var ip_sz int var ip_sz int
var contains bool = true var contains bool = true
/*if contains is true, return bans that *contains* the given value (value is the inner)
else, return bans that are *contained* by the given value (value is the outer)*/
/*the simulated filter is a bit different : if it's not present *or* set to false, specifically exclude records with simulated to true */ // contains == true -> return bans that *contain* the given value (value is the inner)
// contains == false or missing -> return bans *contained* in the given value (value is the outer)
// simulated == true -> include simulated rows
// simulated == false or missing -> exclude simulated rows
if v, ok := filter["simulated"]; ok { if v, ok := filter["simulated"]; ok {
if v[0] == "false" { if v[0] == "false" {
query = query.Where(decision.SimulatedEQ(false)) query = query.Where(decision.SimulatedEQ(false))
@ -33,13 +36,14 @@ func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string]
} else { } else {
query = query.Where(decision.SimulatedEQ(false)) query = query.Where(decision.SimulatedEQ(false))
} }
t := sql.Table(decision.Table)
joinPredicate := make([]*sql.Predicate, 0)
for param, value := range filter { for param, value := range filter {
switch param { switch param {
case "contains": case "contains":
contains, err = strconv.ParseBool(value[0]) contains, err = strconv.ParseBool(value[0])
if err != nil { if err != nil {
return nil, errors.Wrapf(InvalidFilter, "invalid contains value : %s", err) return nil, nil, errors.Wrapf(InvalidFilter, "invalid contains value : %s", err)
} }
case "scopes": case "scopes":
scopes := strings.Split(value[0], ",") scopes := strings.Split(value[0], ",")
@ -64,9 +68,24 @@ func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string]
query = query.Where( query = query.Where(
decision.OriginIn(strings.Split(value[0], ",")...), decision.OriginIn(strings.Split(value[0], ",")...),
) )
origins := strings.Split(value[0], ",")
originsContainsPredicate := make([]*sql.Predicate, 0)
for _, origin := range origins {
pred := sql.EqualFold(t.C(decision.FieldOrigin), origin)
originsContainsPredicate = append(originsContainsPredicate, pred)
}
joinPredicate = append(joinPredicate, sql.Or(originsContainsPredicate...))
case "scenarios_containing": case "scenarios_containing":
predicates := decisionPredicatesFromStr(value[0], decision.ScenarioContainsFold) predicates := decisionPredicatesFromStr(value[0], decision.ScenarioContainsFold)
query = query.Where(decision.Or(predicates...)) query = query.Where(decision.Or(predicates...))
scenarios := strings.Split(value[0], ",")
scenariosContainsPredicate := make([]*sql.Predicate, 0)
for _, scenario := range scenarios {
pred := sql.ContainsFold(t.C(decision.FieldScenario), scenario)
scenariosContainsPredicate = append(scenariosContainsPredicate, pred)
}
joinPredicate = append(joinPredicate, sql.Or(scenariosContainsPredicate...))
case "scenarios_not_containing": case "scenarios_not_containing":
predicates := decisionPredicatesFromStr(value[0], decision.ScenarioContainsFold) predicates := decisionPredicatesFromStr(value[0], decision.ScenarioContainsFold)
query = query.Where(decision.Not( query = query.Where(decision.Not(
@ -74,10 +93,17 @@ func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string]
predicates..., predicates...,
), ),
)) ))
scenarios := strings.Split(value[0], ",")
scenariosContainsPredicate := make([]*sql.Predicate, 0)
for _, scenario := range scenarios {
pred := sql.ContainsFold(t.C(decision.FieldScenario), scenario)
scenariosContainsPredicate = append(scenariosContainsPredicate, sql.Not(pred))
}
joinPredicate = append(joinPredicate, sql.Or(scenariosContainsPredicate...))
case "ip", "range": case "ip", "range":
ip_sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(value[0]) ip_sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(value[0])
if err != nil { if err != nil {
return nil, errors.Wrapf(InvalidIPOrRange, "unable to convert '%s' to int: %s", value[0], err) return nil, nil, errors.Wrapf(InvalidIPOrRange, "unable to convert '%s' to int: %s", value[0], err)
} }
} }
} }
@ -149,9 +175,9 @@ func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string]
)) ))
} }
} else if ip_sz != 0 { } else if ip_sz != 0 {
return nil, errors.Wrapf(InvalidFilter, "Unknown ip size %d", ip_sz) return nil, nil, errors.Wrapf(InvalidFilter, "Unknown ip size %d", ip_sz)
} }
return query, nil return query, joinPredicate, nil
} }
func (c *Client) QueryDecisionWithFilter(filter map[string][]string) ([]*ent.Decision, error) { func (c *Client) QueryDecisionWithFilter(filter map[string][]string) ([]*ent.Decision, error) {
@ -161,7 +187,7 @@ func (c *Client) QueryDecisionWithFilter(filter map[string][]string) ([]*ent.Dec
decisions := c.Ent.Decision.Query(). decisions := c.Ent.Decision.Query().
Where(decision.UntilGTE(time.Now().UTC())) Where(decision.UntilGTE(time.Now().UTC()))
decisions, err = BuildDecisionRequestWithFilter(decisions, filter) decisions, _, err = BuildDecisionRequestWithFilter(decisions, filter)
if err != nil { if err != nil {
return []*ent.Decision{}, err return []*ent.Decision{}, err
} }
@ -185,86 +211,89 @@ func (c *Client) QueryDecisionWithFilter(filter map[string][]string) ([]*ent.Dec
return data, nil return data, nil
} }
// ent translation of https://stackoverflow.com/a/28090544
func longestDecisionForScopeTypeValue(s *sql.Selector) {
t := sql.Table(decision.Table)
s.LeftJoin(t).OnP(sql.And(
sql.ColumnsEQ(
t.C(decision.FieldValue),
s.C(decision.FieldValue),
),
sql.ColumnsEQ(
t.C(decision.FieldType),
s.C(decision.FieldType),
),
sql.ColumnsEQ(
t.C(decision.FieldScope),
s.C(decision.FieldScope),
),
sql.ColumnsGT(
t.C(decision.FieldUntil),
s.C(decision.FieldUntil),
),
))
s.Where(
sql.IsNull(
t.C(decision.FieldUntil),
),
)
}
func (c *Client) QueryAllDecisionsWithFilters(filters map[string][]string) ([]*ent.Decision, error) { func (c *Client) QueryAllDecisionsWithFilters(filters map[string][]string) ([]*ent.Decision, error) {
query := c.Ent.Decision.Query().Where( query := c.Ent.Decision.Query().Where(
decision.UntilGT(time.Now().UTC()), decision.UntilGT(time.Now().UTC()),
longestDecisionForScopeTypeValue,
) )
query, err := BuildDecisionRequestWithFilter(query, filters) query, _, err := BuildDecisionRequestWithFilter(query, filters)
if err != nil { if err != nil {
c.Log.Warningf("QueryAllDecisionsWithFilters : %s", err) c.Log.Warningf("QueryAllDecisionsWithFilters : %s", err)
return []*ent.Decision{}, errors.Wrap(QueryFail, "get all decisions with filters") return []*ent.Decision{}, errors.Wrap(QueryFail, "get all decisions with filters")
} }
data, err := query.All(c.CTX) //Order is *very* important, the dedup assumes that decisions are sorted per IP and per time left
data, err := query.Order(ent.Asc(decision.FieldValue), ent.Desc(decision.FieldUntil)).All(c.CTX)
if err != nil { if err != nil {
c.Log.Warningf("QueryAllDecisionsWithFilters : %s", err) c.Log.Warningf("QueryAllDecisionsWithFilters : %s", err)
return []*ent.Decision{}, errors.Wrap(QueryFail, "get all decisions with filters") return []*ent.Decision{}, errors.Wrap(QueryFail, "get all decisions with filters")
} }
return data, nil return data, nil
} }
func (c *Client) QueryExpiredDecisionsWithFilters(filters map[string][]string) ([]*ent.Decision, error) { func (c *Client) QueryExpiredDecisionsWithFilters(filters map[string][]string) ([]*ent.Decision, error) {
now := time.Now().UTC()
query := c.Ent.Decision.Query().Where( query := c.Ent.Decision.Query().Where(
decision.UntilLT(time.Now().UTC()), decision.UntilLT(time.Now().UTC()),
longestDecisionForScopeTypeValue,
) )
query, err := BuildDecisionRequestWithFilter(query, filters) query, predicates, err := BuildDecisionRequestWithFilter(query, filters)
if err != nil { if err != nil {
c.Log.Warningf("QueryExpiredDecisionsWithFilters : %s", err) c.Log.Warningf("QueryExpiredDecisionsWithFilters : %s", err)
return []*ent.Decision{}, errors.Wrap(QueryFail, "get expired decisions with filters") return []*ent.Decision{}, errors.Wrap(QueryFail, "get expired decisions with filters")
} }
data, err := query.All(c.CTX) query = query.Where(func(s *sql.Selector) {
t := sql.Table(decision.Table)
subQuery := sql.Select(t.C(decision.FieldValue)).From(t).Where(sql.GT(t.C(decision.FieldUntil), now))
for _, predicate := range predicates {
subQuery.Where(predicate)
}
s.Where(
sql.NotIn(
s.C(decision.FieldValue),
subQuery,
),
)
})
data, err := query.Order(ent.Asc(decision.FieldValue), ent.Desc(decision.FieldUntil)).All(c.CTX)
if err != nil { if err != nil {
c.Log.Warningf("QueryExpiredDecisionsWithFilters : %s", err) c.Log.Warningf("QueryExpiredDecisionsWithFilters : %s", err)
return []*ent.Decision{}, errors.Wrap(QueryFail, "expired decisions") return []*ent.Decision{}, errors.Wrap(QueryFail, "expired decisions")
} }
return data, nil return data, nil
} }
func (c *Client) QueryExpiredDecisionsSinceWithFilters(since time.Time, filters map[string][]string) ([]*ent.Decision, error) { func (c *Client) QueryExpiredDecisionsSinceWithFilters(since time.Time, filters map[string][]string) ([]*ent.Decision, error) {
now := time.Now().UTC()
query := c.Ent.Decision.Query().Where( query := c.Ent.Decision.Query().Where(
decision.UntilLT(time.Now().UTC()), decision.UntilLT(now),
decision.UntilGT(since), decision.UntilGT(since),
longestDecisionForScopeTypeValue,
) )
query, err := BuildDecisionRequestWithFilter(query, filters) query, predicates, err := BuildDecisionRequestWithFilter(query, filters)
if err != nil { if err != nil {
c.Log.Warningf("QueryExpiredDecisionsSinceWithFilters : %s", err) c.Log.Warningf("QueryExpiredDecisionsSinceWithFilters : %s", err)
return []*ent.Decision{}, errors.Wrap(QueryFail, "expired decisions with filters") return []*ent.Decision{}, errors.Wrap(QueryFail, "expired decisions with filters")
} }
data, err := query.All(c.CTX) query = query.Where(func(s *sql.Selector) {
t := sql.Table(decision.Table)
subQuery := sql.Select(t.C(decision.FieldValue)).From(t).Where(sql.GT(t.C(decision.FieldUntil), now))
for _, predicate := range predicates {
subQuery.Where(predicate)
}
s.Where(
sql.NotIn(
s.C(decision.FieldValue),
subQuery,
),
)
})
data, err := query.Order(ent.Asc(decision.FieldValue), ent.Desc(decision.FieldUntil)).All(c.CTX)
if err != nil { if err != nil {
c.Log.Warningf("QueryExpiredDecisionsSinceWithFilters : %s", err) c.Log.Warningf("QueryExpiredDecisionsSinceWithFilters : %s", err)
return []*ent.Decision{}, errors.Wrap(QueryFail, "expired decisions with filters") return []*ent.Decision{}, errors.Wrap(QueryFail, "expired decisions with filters")
@ -277,14 +306,15 @@ func (c *Client) QueryNewDecisionsSinceWithFilters(since time.Time, filters map[
query := c.Ent.Decision.Query().Where( query := c.Ent.Decision.Query().Where(
decision.CreatedAtGT(since), decision.CreatedAtGT(since),
decision.UntilGT(time.Now().UTC()), decision.UntilGT(time.Now().UTC()),
longestDecisionForScopeTypeValue,
) )
query, err := BuildDecisionRequestWithFilter(query, filters) query, _, err := BuildDecisionRequestWithFilter(query, filters)
if err != nil { if err != nil {
c.Log.Warningf("QueryNewDecisionsSinceWithFilters : %s", err) c.Log.Warningf("BuildDecisionRequestWithFilter : %s", err)
return []*ent.Decision{}, errors.Wrapf(QueryFail, "new decisions since '%s'", since.String()) return []*ent.Decision{}, errors.Wrap(QueryFail, "expired decisions with filters")
} }
data, err := query.All(c.CTX)
//Order is *very* important, the dedup assumes that decisions are sorted per IP and per time left
data, err := query.Order(ent.Asc(decision.FieldValue), ent.Desc(decision.FieldUntil)).All(c.CTX)
if err != nil { if err != nil {
c.Log.Warningf("QueryNewDecisionsSinceWithFilters : %s", err) c.Log.Warningf("QueryNewDecisionsSinceWithFilters : %s", err)
return []*ent.Decision{}, errors.Wrapf(QueryFail, "new decisions since '%s'", since.String()) return []*ent.Decision{}, errors.Wrapf(QueryFail, "new decisions since '%s'", since.String())
@ -521,17 +551,17 @@ func (c *Client) SoftDeleteDecisionsWithFilter(filter map[string][]string) (stri
} }
//SoftDeleteDecisionByID set the expiration of a decision to now() //SoftDeleteDecisionByID set the expiration of a decision to now()
func (c *Client) SoftDeleteDecisionByID(decisionID int) error { func (c *Client) SoftDeleteDecisionByID(decisionID int) (int, error) {
nbUpdated, err := c.Ent.Decision.Update().Where(decision.IDEQ(decisionID)).SetUntil(time.Now().UTC()).Save(c.CTX) nbUpdated, err := c.Ent.Decision.Update().Where(decision.IDEQ(decisionID)).SetUntil(time.Now().UTC()).Save(c.CTX)
if err != nil || nbUpdated == 0 { if err != nil || nbUpdated == 0 {
c.Log.Warningf("SoftDeleteDecisionByID : %v (nb soft deleted: %d)", err, nbUpdated) c.Log.Warningf("SoftDeleteDecisionByID : %v (nb soft deleted: %d)", err, nbUpdated)
return errors.Wrapf(DeleteFail, "decision with id '%d' doesn't exist", decisionID) return 0, errors.Wrapf(DeleteFail, "decision with id '%d' doesn't exist", decisionID)
} }
if nbUpdated == 0 { if nbUpdated == 0 {
return ItemNotFound return 0, ItemNotFound
} }
return nil return nbUpdated, nil
} }
func decisionPredicatesFromStr(s string, predicateFunc func(string) predicate.Decision) []predicate.Decision { func decisionPredicatesFromStr(s string, predicateFunc func(string) predicate.Decision) []predicate.Decision {

233
tests/bats/99_lapi-stream-mode-scenario.bats Normal file
View file

@ -0,0 +1,233 @@
#!/usr/bin/env bats
# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
set -u
setup_file() {
load "../lib/setup_file.sh"
./instance-data load
./instance-crowdsec start
API_KEY=$(cscli bouncers add testbouncer -o raw)
export API_KEY
CROWDSEC_API_URL="http://localhost:8080"
export CROWDSEC_API_URL
}
teardown_file() {
load "../lib/teardown_file.sh"
}
setup() {
load "../lib/setup.sh"
skip
}
#----------
api() {
URI="$1"
curl -s -H "X-Api-Key:${API_KEY}" "${CROWDSEC_API_URL}${URI}"
}
output_new_decisions() {
jq -c '.new | map(select(.origin!="CAPI")) | .[] | del(.id) | (.. | .duration?) |= capture("(?<d>[[:digit:]]+h[[:digit:]]+m)").d' <(output) | sort
}
@test "${FILE} adding decisions with different duration, scenario, origin" {
# origin: test
run -0 cscli decisions add -i 127.0.0.1 -d 1h -R crowdsecurity/test
./instance-crowdsec stop
run -0 ./instance-db exec_sql "update decisions set origin='test' where origin='cscli'"
./instance-crowdsec start
run -0 cscli decisions add -i 127.0.0.1 -d 3h -R crowdsecurity/ssh_bf
./instance-crowdsec stop
run -0 ./instance-db exec_sql "update decisions set origin='another_origin' where origin='cscli'"
./instance-crowdsec start
run -0 cscli decisions add -i 127.0.0.1 -d 5h -R crowdsecurity/longest
run -0 cscli decisions add -i 127.0.0.2 -d 3h -R crowdsecurity/test
run -0 cscli decisions add -i 127.0.0.2 -d 3h -R crowdsecurity/ssh_bf
run -0 cscli decisions add -i 127.0.0.2 -d 1h -R crowdsecurity/ssh_bf
./instance-crowdsec stop
run -0 ./instance-db exec_sql "update decisions set origin='test' where origin='cscli'"
./instance-crowdsec start
# origin: another_origin
run -0 cscli decisions add -i 127.0.0.2 -d 2h -R crowdsecurity/test
./instance-crowdsec stop
run -0 ./instance-db exec_sql "update decisions set origin='another_origin' where origin='cscli'"
./instance-crowdsec start
}
@test "${FILE} test startup" {
run -0 api "/v1/decisions/stream?startup=true"
run -0 output_new_decisions
assert_output - <<-EOT
{"duration":"2h59m","origin":"test","scenario":"crowdsecurity/test","scope":"Ip","type":"ban","value":"127.0.0.2"}
{"duration":"4h59m","origin":"test","scenario":"crowdsecurity/longest","scope":"Ip","type":"ban","value":"127.0.0.1"}
EOT
}
@test "${FILE} test startup with scenarios containing" {
run -0 api "/v1/decisions/stream?startup=true&scenarios_containing=ssh_bf"
run -0 output_new_decisions
assert_output - <<-EOT
{"duration":"2h59m","origin":"another_origin","scenario":"crowdsecurity/ssh_bf","scope":"Ip","type":"ban","value":"127.0.0.1"}
{"duration":"2h59m","origin":"test","scenario":"crowdsecurity/ssh_bf","scope":"Ip","type":"ban","value":"127.0.0.2"}
EOT
}
@test "${FILE} test startup with multiple scenarios containing" {
run -0 api "/v1/decisions/stream?startup=true&scenarios_containing=ssh_bf,test"
run -0 output_new_decisions
assert_output - <<-EOT
{"duration":"2h59m","origin":"another_origin","scenario":"crowdsecurity/ssh_bf","scope":"Ip","type":"ban","value":"127.0.0.1"}
{"duration":"2h59m","origin":"test","scenario":"crowdsecurity/test","scope":"Ip","type":"ban","value":"127.0.0.2"}
EOT
}
@test "${FILE} test startup with unknown scenarios containing" {
run -0 api "/v1/decisions/stream?startup=true&scenarios_containing=unknown"
assert_output '{"deleted":null,"new":null}'
}
@test "${FILE} test startup with scenarios containing and not containing" {
run -0 api "/v1/decisions/stream?startup=true&scenarios_containing=test&scenarios_not_containing=ssh_bf"
run -0 output_new_decisions
assert_output - <<-EOT
{"duration":"2h59m","origin":"test","scenario":"crowdsecurity/test","scope":"Ip","type":"ban","value":"127.0.0.2"}
{"origin":"test","scenario":"crowdsecurity/test","scope":"Ip","type":"ban","value":"127.0.0.1"}
EOT
}
@test "${FILE} test startup with scenarios containing and not containing 2" {
run -0 api "/v1/decisions/stream?startup=true&scenarios_containing=longest&scenarios_not_containing=ssh_bf,test"
run -0 output_new_decisions
assert_output - <<-EOT
{"duration":"4h59m","origin":"test","scenario":"crowdsecurity/longest","scope":"Ip","type":"ban","value":"127.0.0.1"}
EOT
}
@test "${FILE} test startup with scenarios not containing" {
run -0 api "/v1/decisions/stream?startup=true&scenarios_not_containing=ssh_bf"
run -0 output_new_decisions
assert_output - <<-EOT
{"duration":"2h59m","origin":"test","scenario":"crowdsecurity/test","scope":"Ip","type":"ban","value":"127.0.0.2"}
{"duration":"4h59m","origin":"test","scenario":"crowdsecurity/longest","scope":"Ip","type":"ban","value":"127.0.0.1"}
EOT
}
@test "${FILE} test startup with multiple scenarios not containing" {
run -0 api "/v1/decisions/stream?startup=true&scenarios_not_containing=ssh_bf,test"
run -0 output_new_decisions
assert_output - <<-EOT
{"duration":"4h59m","origin":"test","scenario":"crowdsecurity/longest","scope":"Ip","type":"ban","value":"127.0.0.1"}
EOT
}
@test "${FILE} test startup with origins parameter" {
run -0 api "/v1/decisions/stream?startup=true&origins=another_origin"
run -0 output_new_decisions
assert_output - <<-EOT
{"duration":"1h59m","origin":"another_origin","scenario":"crowdsecurity/test","scope":"Ip","type":"ban","value":"127.0.0.2"}
{"duration":"2h59m","origin":"another_origin","scenario":"crowdsecurity/ssh_bf","scope":"Ip","type":"ban","value":"127.0.0.1"}
EOT
}
@test "${FILE} test startup with multiple origins parameter" {
run -0 api "/v1/decisions/stream?startup=true&origins=another_origin,test"
run -0 output_new_decisions
assert_output - <<-EOT
{"duration":"2h59m","origin":"test","scenario":"crowdsecurity/test","scope":"Ip","type":"ban","value":"127.0.0.2"}
{"duration":"4h59m","origin":"test","scenario":"crowdsecurity/longest","scope":"Ip","type":"ban","value":"127.0.0.1"}
EOT
}
@test "${FILE} test startup with unknown origins" {
run -0 api "/v1/decisions/stream?startup=true&origins=unknown"
assert_output '{"deleted":null,"new":null}'
}
#@test "${FILE} delete decision 3 (127.0.0.1)" {
#
# {
# TestName: "delete decisions 3 (127.0.0.1)",
# Method: "DELETE",
# Route: "/v1/decisions/3",
# CheckCodeOnly: true,
# Code: 200,
# LenNew: 0,
# LenDeleted: 0,
# AuthType: PASSWORD,
# DelChecks: []DecisionCheck{},
# NewChecks: []DecisionCheck{},
# TestName: "check that 127.0.0.1 is not in deleted IP",
# Method: "GET",
# Route: "/v1/decisions/stream?startup=true",
# CheckCodeOnly: false,
# Code: 200,
# LenNew: 2,
# LenDeleted: 0,
# AuthType: APIKEY,
# DelChecks: []DecisionCheck{},
# NewChecks: []DecisionCheck{},
# },
# {
# TestName: "delete decisions 2 (127.0.0.1)",
# Method: "DELETE",
# Route: "/v1/decisions/2",
# CheckCodeOnly: true,
# Code: 200,
# LenNew: 0,
# LenDeleted: 0,
# AuthType: PASSWORD,
# DelChecks: []DecisionCheck{},
# NewChecks: []DecisionCheck{},
# },
# {
# TestName: "check that 127.0.0.1 is not in deleted IP",
# Method: "GET",
# Route: "/v1/decisions/stream?startup=true",
# CheckCodeOnly: false,
# Code: 200,
# LenNew: 2,
# LenDeleted: 0,
# AuthType: APIKEY,
# DelChecks: []DecisionCheck{},
# NewChecks: []DecisionCheck{},
# },
# {
# TestName: "delete decisions 1 (127.0.0.1)",
# Method: "DELETE",
# Route: "/v1/decisions/1",
# CheckCodeOnly: true,
# Code: 200,
# LenNew: 0,
# LenDeleted: 0,
# AuthType: PASSWORD,
# DelChecks: []DecisionCheck{},
# NewChecks: []DecisionCheck{},
# },
# TestName: "127.0.0.1 should be in deleted now",
# Method: "GET",
# Route: "/v1/decisions/stream?startup=true",
# CheckCodeOnly: false,
# Code: 200,
# LenNew: 1,
# LenDeleted: 1,
# AuthType: APIKEY,
# DelChecks: []DecisionCheck{
# {
# ID: int64(1),
# Origin: "test",
# Scenario: "crowdsecurity/test",
# Value: "127.0.0.1",
# Duration: "-", // we check that the time is negative
# },
# },
# NewChecks: []DecisionCheck{},
# },
#}

8
tests/lib/db/instance-mysql
View file

@ -115,7 +115,13 @@ case "$1" in
;; ;;
exec_sql) exec_sql)
shift shift
exec_sql "$@" #
# This command is meant to run a query against the the crowdsec database.
# The exec_sql() function is more generic and is also used for database setup and backups.
#
# For this reason, we select the database here.
#
exec_sql "use crowdsec_test; $@"
;; ;;
*) *)
about about

4
tests/lib/setup_file.sh
View file

@ -4,6 +4,10 @@
# https://github.com/bats-core/bats-core/blob/master/docs/source/warnings/BW02.rst # https://github.com/bats-core/bats-core/blob/master/docs/source/warnings/BW02.rst
bats_require_minimum_version 1.5.0 bats_require_minimum_version 1.5.0
# this should have effect globally, for all tests
# https://github.com/bats-core/bats-core/blob/master/docs/source/warnings/BW02.rst
bats_require_minimum_version 1.5.0
debug() { debug() {
echo 'exec 1<&-; exec 2<&-; exec 1>&3; exec 2>&1' echo 'exec 1<&-; exec 2<&-; exec 1>&3; exec 2>&1'
} }