mirror of
https://github.com/crowdsecurity/crowdsec.git
synced 2025-05-11 20:36:12 +02:00
Thibault "bui" Koechlin
GitHub
parent
5f339ab312
commit
dbb420f79e
563 changed files with 64363 additions and 10714 deletions
|
|
@ -1,135 +1,251 @@
|
|||
package leakybucket
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/models"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/go-openapi/strfmt"
|
||||
"github.com/pkg/errors"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/antonmedv/expr"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
|
||||
)
|
||||
|
||||
func FormatOverflow(l *Leaky, queue *Queue) types.SignalOccurence {
|
||||
var am string
|
||||
//SourceFromEvent extracts and formats a valid models.Source object from an Event
|
||||
func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, error) {
|
||||
src := models.Source{}
|
||||
srcs := make(map[string]models.Source)
|
||||
|
||||
l.logger.Debugf("Overflow (start: %s, end: %s)", l.First_ts, l.Ovflw_ts)
|
||||
|
||||
sig := types.SignalOccurence{
|
||||
Scenario: l.Name,
|
||||
Bucket_id: l.Uuid,
|
||||
Alert_message: am,
|
||||
Start_at: l.First_ts,
|
||||
Stop_at: l.Ovflw_ts,
|
||||
Events_count: l.Total_count,
|
||||
Capacity: l.Capacity,
|
||||
Reprocess: l.Reprocess,
|
||||
Leak_speed: l.Leakspeed,
|
||||
MapKey: l.Mapkey,
|
||||
Sources: make(map[string]types.Source),
|
||||
Labels: l.BucketConfig.Labels,
|
||||
if evt.Type == types.OVFLW {
|
||||
return evt.Overflow.Sources, nil
|
||||
}
|
||||
switch leaky.scopeType.Scope {
|
||||
case types.Range, types.Ip:
|
||||
if v, ok := evt.Meta["source_ip"]; ok {
|
||||
if net.ParseIP(v) == nil {
|
||||
return srcs, fmt.Errorf("scope is %s but '%s' isn't a valid ip", leaky.scopeType.Scope, v)
|
||||
} else {
|
||||
src.IP = v
|
||||
}
|
||||
} else {
|
||||
return srcs, fmt.Errorf("scope is %s but Meta[source_ip] doesn't exist", leaky.scopeType.Scope)
|
||||
}
|
||||
|
||||
src.Scope = &leaky.scopeType.Scope
|
||||
if v, ok := evt.Enriched["ASNumber"]; ok {
|
||||
src.AsNumber = v
|
||||
}
|
||||
if v, ok := evt.Enriched["IsoCode"]; ok {
|
||||
src.Cn = v
|
||||
}
|
||||
if v, ok := evt.Enriched["ASNOrg"]; ok {
|
||||
src.AsName = v
|
||||
}
|
||||
if v, ok := evt.Enriched["Latitude"]; ok {
|
||||
l, err := strconv.ParseFloat(v, 32)
|
||||
if err != nil {
|
||||
log.Warningf("bad latitude %s : %s", v, err)
|
||||
}
|
||||
src.Latitude = float32(l)
|
||||
}
|
||||
if v, ok := evt.Enriched["Longitude"]; ok {
|
||||
l, err := strconv.ParseFloat(v, 32)
|
||||
if err != nil {
|
||||
log.Warningf("bad longitude %s : %s", v, err)
|
||||
}
|
||||
src.Longitude = float32(l)
|
||||
}
|
||||
if v, ok := evt.Meta["SourceRange"]; ok && v != "" {
|
||||
_, ipNet, err := net.ParseCIDR(v)
|
||||
if err != nil {
|
||||
return srcs, fmt.Errorf("Declared range %s of %s can't be parsed", v, src.IP)
|
||||
} else if ipNet != nil {
|
||||
src.Range = ipNet.String()
|
||||
leaky.logger.Tracef("Valid range from %s : %s", src.IP, src.Range)
|
||||
}
|
||||
}
|
||||
if leaky.scopeType.Scope == types.Ip {
|
||||
src.Value = &src.IP
|
||||
} else if leaky.scopeType.Scope == types.Range {
|
||||
src.Value = &src.Range
|
||||
}
|
||||
srcs[src.IP] = src
|
||||
default:
|
||||
if leaky.scopeType.RunTimeFilter != nil {
|
||||
retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, exprhelpers.GetExprEnv(map[string]interface{}{"evt": &evt}))
|
||||
if err != nil {
|
||||
return srcs, errors.Wrapf(err, "while running scope filter")
|
||||
}
|
||||
|
||||
value, ok := retValue.(string)
|
||||
if !ok {
|
||||
value = ""
|
||||
}
|
||||
src.Value = &value
|
||||
src.Scope = new(string)
|
||||
*src.Scope = leaky.scopeType.Scope
|
||||
srcs[*src.Value] = src
|
||||
log.Debugf("source[%s] - %s = %s", leaky.Name, leaky.scopeType.Scope, *src.Value)
|
||||
} else {
|
||||
return srcs, fmt.Errorf("empty scope information")
|
||||
}
|
||||
}
|
||||
return srcs, nil
|
||||
}
|
||||
|
||||
//EventsFromQueue iterates the queue to collect & prepare meta-datas from alert
|
||||
func EventsFromQueue(queue *Queue) []*models.Event {
|
||||
|
||||
events := []*models.Event{}
|
||||
|
||||
for _, evt := range queue.Queue {
|
||||
//either it's a collection of logs, or a collection of past overflows being reprocessed.
|
||||
//one overflow can have multiple sources for example
|
||||
if evt.Type == types.LOG {
|
||||
if _, ok := evt.Meta["source_ip"]; !ok {
|
||||
continue
|
||||
}
|
||||
source_ip := evt.Meta["source_ip"]
|
||||
if _, ok := sig.Sources[source_ip]; !ok {
|
||||
src := types.Source{}
|
||||
src.Ip = net.ParseIP(source_ip)
|
||||
if v, ok := evt.Enriched["ASNNumber"]; ok {
|
||||
src.AutonomousSystemNumber = v
|
||||
}
|
||||
if v, ok := evt.Enriched["IsoCode"]; ok {
|
||||
src.Country = v
|
||||
}
|
||||
if v, ok := evt.Enriched["ASNOrg"]; ok {
|
||||
src.AutonomousSystemOrganization = v
|
||||
}
|
||||
if v, ok := evt.Enriched["Latitude"]; ok {
|
||||
src.Latitude, _ = strconv.ParseFloat(v, 32)
|
||||
}
|
||||
if v, ok := evt.Enriched["Longitude"]; ok {
|
||||
src.Longitude, _ = strconv.ParseFloat(v, 32)
|
||||
}
|
||||
if v, ok := evt.Meta["SourceRange"]; ok {
|
||||
_, ipNet, err := net.ParseCIDR(v)
|
||||
if err != nil {
|
||||
l.logger.Errorf("Declared range %s of %s can't be parsed", v, src.Ip.String())
|
||||
} else if ipNet != nil {
|
||||
src.Range = *ipNet
|
||||
l.logger.Tracef("Valid range from %s : %s", src.Ip.String(), src.Range.String())
|
||||
}
|
||||
}
|
||||
sig.Sources[source_ip] = src
|
||||
if sig.Source == nil {
|
||||
sig.Source = &src
|
||||
sig.Source_ip = src.Ip.String()
|
||||
sig.Source_AutonomousSystemNumber = src.AutonomousSystemNumber
|
||||
sig.Source_AutonomousSystemOrganization = src.AutonomousSystemOrganization
|
||||
sig.Source_Country = src.Country
|
||||
sig.Source_range = src.Range.String()
|
||||
sig.Source_Latitude = src.Latitude
|
||||
sig.Source_Longitude = src.Longitude
|
||||
}
|
||||
}
|
||||
} else if evt.Type == types.OVFLW {
|
||||
for _, src := range evt.Overflow.Sources {
|
||||
if _, ok := sig.Sources[src.Ip.String()]; !ok {
|
||||
sig.Sources[src.Ip.String()] = src
|
||||
if sig.Source == nil {
|
||||
l.logger.Tracef("populating overflow with source : %+v", src)
|
||||
src := src //src will be reused, copy before giving pointer
|
||||
sig.Source = &src
|
||||
sig.Source_ip = src.Ip.String()
|
||||
sig.Source_AutonomousSystemNumber = src.AutonomousSystemNumber
|
||||
sig.Source_AutonomousSystemOrganization = src.AutonomousSystemOrganization
|
||||
sig.Source_Country = src.Country
|
||||
sig.Source_range = src.Range.String()
|
||||
sig.Source_Latitude = src.Latitude
|
||||
sig.Source_Longitude = src.Longitude
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
strret, err := json.Marshal(evt.Meta)
|
||||
if err != nil {
|
||||
l.logger.Errorf("failed to marshal ret : %v", err)
|
||||
if evt.Meta == nil {
|
||||
continue
|
||||
}
|
||||
if sig.Source != nil {
|
||||
sig.Events_sequence = append(sig.Events_sequence, types.EventSequence{
|
||||
Source: *sig.Source,
|
||||
Source_ip: sig.Source_ip,
|
||||
Source_AutonomousSystemNumber: sig.Source.AutonomousSystemNumber,
|
||||
Source_AutonomousSystemOrganization: sig.Source.AutonomousSystemOrganization,
|
||||
Source_Country: sig.Source.Country,
|
||||
Serialized: string(strret),
|
||||
Time: l.First_ts})
|
||||
meta := models.Meta{}
|
||||
for k, v := range evt.Meta {
|
||||
subMeta := models.MetaItems0{Key: k, Value: v}
|
||||
meta = append(meta, &subMeta)
|
||||
}
|
||||
|
||||
/*check which date to use*/
|
||||
ovflwEvent := models.Event{
|
||||
Meta: meta,
|
||||
}
|
||||
//either MarshaledTime is present and is extracted from log
|
||||
if evt.MarshaledTime != "" {
|
||||
ovflwEvent.Timestamp = &evt.MarshaledTime
|
||||
} else if !evt.Time.IsZero() { //or .Time has been set during parse as time.Now()
|
||||
ovflwEvent.Timestamp = new(string)
|
||||
raw, err := evt.Time.MarshalText()
|
||||
if err != nil {
|
||||
log.Warningf("while marshaling time '%s' : %s", evt.Time.String(), err)
|
||||
} else {
|
||||
*ovflwEvent.Timestamp = string(raw)
|
||||
}
|
||||
} else {
|
||||
l.logger.Warningf("Event without source ?!")
|
||||
log.Warningf("Event has no parsed time, no runtime timestamp")
|
||||
}
|
||||
|
||||
events = append(events, &ovflwEvent)
|
||||
}
|
||||
return events
|
||||
}
|
||||
|
||||
//alertFormatSource iterates over the queue to collect sources
|
||||
func alertFormatSource(leaky *Leaky, queue *Queue) (map[string]models.Source, string, error) {
|
||||
var sources map[string]models.Source = make(map[string]models.Source)
|
||||
var source_type string
|
||||
|
||||
log.Debugf("Formatting (%s) - scope Info : scope_type:%s / scope_filter:%s", leaky.Name, leaky.scopeType.Scope, leaky.scopeType.Filter)
|
||||
|
||||
for _, evt := range queue.Queue {
|
||||
srcs, err := SourceFromEvent(evt, leaky)
|
||||
if err != nil {
|
||||
return nil, "", errors.Wrapf(err, "while extracting scope from bucket %s", leaky.Name)
|
||||
}
|
||||
for key, src := range srcs {
|
||||
if source_type == types.Undefined {
|
||||
source_type = *src.Scope
|
||||
}
|
||||
if *src.Scope != source_type {
|
||||
return nil, "",
|
||||
fmt.Errorf("event has multiple source types : %s != %s", *src.Scope, source_type)
|
||||
}
|
||||
sources[key] = src
|
||||
}
|
||||
}
|
||||
return sources, source_type, nil
|
||||
}
|
||||
|
||||
if len(sig.Sources) > 1 {
|
||||
am = fmt.Sprintf("%d IPs", len(sig.Sources))
|
||||
} else if len(sig.Sources) == 1 {
|
||||
if sig.Source != nil {
|
||||
am = sig.Source.Ip.String()
|
||||
} else {
|
||||
am = "??"
|
||||
//NewAlert will generate a RuntimeAlert and its APIAlert(s) from a bucket that overflowed
|
||||
func NewAlert(leaky *Leaky, queue *Queue) (types.RuntimeAlert, error) {
|
||||
|
||||
var runtimeAlert types.RuntimeAlert
|
||||
|
||||
leaky.logger.Tracef("Overflow (start: %s, end: %s)", leaky.First_ts, leaky.Ovflw_ts)
|
||||
/*
|
||||
Craft the models.Alert that is going to be duplicated for each source
|
||||
*/
|
||||
start_at, err := leaky.First_ts.MarshalText()
|
||||
if err != nil {
|
||||
log.Warningf("failed to marshal start ts %s : %s", leaky.First_ts.String(), err)
|
||||
}
|
||||
stop_at, err := leaky.Ovflw_ts.MarshalText()
|
||||
if err != nil {
|
||||
log.Warningf("failed to marshal ovflw ts %s : %s", leaky.First_ts.String(), err)
|
||||
}
|
||||
capacity := int32(leaky.Capacity)
|
||||
EventsCount := int32(leaky.Total_count)
|
||||
leakSpeed := leaky.Leakspeed.String()
|
||||
startAt := string(start_at)
|
||||
stopAt := string(stop_at)
|
||||
apiAlert := models.Alert{
|
||||
Scenario: &leaky.Name,
|
||||
ScenarioHash: &leaky.hash,
|
||||
ScenarioVersion: &leaky.scenarioVersion,
|
||||
Capacity: &capacity,
|
||||
EventsCount: &EventsCount,
|
||||
Leakspeed: &leakSpeed,
|
||||
Message: new(string),
|
||||
StartAt: &startAt,
|
||||
StopAt: &stopAt,
|
||||
Simulated: &leaky.Simulated,
|
||||
}
|
||||
if leaky.BucketConfig == nil {
|
||||
return runtimeAlert, fmt.Errorf("leaky.BucketConfig is nil")
|
||||
}
|
||||
|
||||
//give information about the bucket
|
||||
runtimeAlert.Mapkey = leaky.Mapkey
|
||||
|
||||
//Get the sources from Leaky/Queue
|
||||
sources, source_scope, err := alertFormatSource(leaky, queue)
|
||||
if err != nil {
|
||||
return runtimeAlert, errors.Wrap(err, "unable to collect sources from bucket")
|
||||
}
|
||||
runtimeAlert.Sources = sources
|
||||
//Include source info in format string
|
||||
sourceStr := ""
|
||||
if len(sources) > 1 {
|
||||
sourceStr = fmt.Sprintf("%d Sources on scope.", len(sources))
|
||||
} else if len(sources) == 1 {
|
||||
for k, _ := range sources {
|
||||
sourceStr = k
|
||||
break
|
||||
}
|
||||
} else {
|
||||
am = "UNKNOWN"
|
||||
sourceStr = "UNKNOWN"
|
||||
}
|
||||
*apiAlert.Message = fmt.Sprintf("%s %s performed '%s' (%d events over %s) at %s", source_scope, sourceStr, leaky.Name, leaky.Total_count, leaky.Ovflw_ts.Sub(leaky.First_ts), leaky.Last_ts)
|
||||
//Get the events from Leaky/Queue
|
||||
apiAlert.Events = EventsFromQueue(queue)
|
||||
|
||||
//Loop over the Sources and generate appropriate number of ApiAlerts
|
||||
for _, srcValue := range sources {
|
||||
newApiAlert := apiAlert
|
||||
srcCopy := srcValue
|
||||
newApiAlert.Source = &srcCopy
|
||||
if v, ok := leaky.BucketConfig.Labels["remediation"]; ok && v == "true" {
|
||||
newApiAlert.Remediation = true
|
||||
}
|
||||
|
||||
if err := newApiAlert.Validate(strfmt.Default); err != nil {
|
||||
log.Errorf("Generated alerts isn't valid")
|
||||
log.Errorf("->%s", spew.Sdump(newApiAlert))
|
||||
log.Fatalf("error : %s", err)
|
||||
}
|
||||
runtimeAlert.APIAlerts = append(runtimeAlert.APIAlerts, newApiAlert)
|
||||
}
|
||||
|
||||
am += fmt.Sprintf(" performed '%s' (%d events over %s) at %s", l.Name, l.Total_count, l.Ovflw_ts.Sub(l.First_ts), l.Ovflw_ts)
|
||||
sig.Alert_message = am
|
||||
return sig
|
||||
runtimeAlert.Alert = &runtimeAlert.APIAlerts[0]
|
||||
if leaky.Reprocess {
|
||||
runtimeAlert.Reprocess = true
|
||||
}
|
||||
return runtimeAlert, nil
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue