mirror of
https://github.com/crowdsecurity/crowdsec.git
synced 2025-05-10 20:05:55 +02:00
394 lines
12 KiB
Go
394 lines
12 KiB
Go
package appsecacquisition
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
log "github.com/sirupsen/logrus"
|
|
"gopkg.in/tomb.v2"
|
|
|
|
"github.com/crowdsecurity/coraza/v3"
|
|
corazatypes "github.com/crowdsecurity/coraza/v3/types"
|
|
|
|
// load body processors via init()
|
|
_ "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/appsec/bodyprocessors"
|
|
"github.com/crowdsecurity/crowdsec/pkg/appsec"
|
|
"github.com/crowdsecurity/crowdsec/pkg/types"
|
|
)
|
|
|
|
// that's the runtime structure of the Application security engine as seen from the acquis
|
|
type AppsecRunner struct {
|
|
outChan chan types.Event
|
|
inChan chan appsec.ParsedRequest
|
|
UUID string
|
|
AppsecRuntime *appsec.AppsecRuntimeConfig //this holds the actual appsec runtime config, rules, remediations, hooks etc.
|
|
AppsecInbandEngine coraza.WAF
|
|
AppsecOutbandEngine coraza.WAF
|
|
Labels map[string]string
|
|
logger *log.Entry
|
|
}
|
|
|
|
func (r *AppsecRunner) MergeDedupRules(collections []appsec.AppsecCollection, logger *log.Entry) string {
|
|
var rulesArr []string
|
|
dedupRules := make(map[string]struct{})
|
|
|
|
for _, collection := range collections {
|
|
for _, rule := range collection.Rules {
|
|
if _, ok := dedupRules[rule]; !ok {
|
|
rulesArr = append(rulesArr, rule)
|
|
dedupRules[rule] = struct{}{}
|
|
} else {
|
|
logger.Debugf("Discarding duplicate rule : %s", rule)
|
|
}
|
|
}
|
|
}
|
|
if len(rulesArr) != len(dedupRules) {
|
|
logger.Warningf("%d rules were discarded as they were duplicates", len(rulesArr)-len(dedupRules))
|
|
}
|
|
|
|
return strings.Join(rulesArr, "\n")
|
|
}
|
|
|
|
func (r *AppsecRunner) Init(datadir string) error {
|
|
var err error
|
|
fs := os.DirFS(datadir)
|
|
|
|
inBandLogger := r.logger.Dup().WithField("band", "inband")
|
|
outBandLogger := r.logger.Dup().WithField("band", "outband")
|
|
|
|
//While loading rules, we dedup rules based on their content, while keeping the order
|
|
inBandRules := r.MergeDedupRules(r.AppsecRuntime.InBandRules, inBandLogger)
|
|
outOfBandRules := r.MergeDedupRules(r.AppsecRuntime.OutOfBandRules, outBandLogger)
|
|
|
|
//setting up inband engine
|
|
inbandCfg := coraza.NewWAFConfig().WithDirectives(inBandRules).WithRootFS(fs).WithDebugLogger(appsec.NewCrzLogger(inBandLogger))
|
|
if !r.AppsecRuntime.Config.InbandOptions.DisableBodyInspection {
|
|
inbandCfg = inbandCfg.WithRequestBodyAccess()
|
|
} else {
|
|
log.Warningf("Disabling body inspection, Inband rules will not be able to match on body's content.")
|
|
}
|
|
if r.AppsecRuntime.Config.InbandOptions.RequestBodyInMemoryLimit != nil {
|
|
inbandCfg = inbandCfg.WithRequestBodyInMemoryLimit(*r.AppsecRuntime.Config.InbandOptions.RequestBodyInMemoryLimit)
|
|
}
|
|
r.AppsecInbandEngine, err = coraza.NewWAF(inbandCfg)
|
|
if err != nil {
|
|
return fmt.Errorf("unable to initialize inband engine : %w", err)
|
|
}
|
|
|
|
//setting up outband engine
|
|
outbandCfg := coraza.NewWAFConfig().WithDirectives(outOfBandRules).WithRootFS(fs).WithDebugLogger(appsec.NewCrzLogger(outBandLogger))
|
|
if !r.AppsecRuntime.Config.OutOfBandOptions.DisableBodyInspection {
|
|
outbandCfg = outbandCfg.WithRequestBodyAccess()
|
|
} else {
|
|
log.Warningf("Disabling body inspection, Out of band rules will not be able to match on body's content.")
|
|
}
|
|
if r.AppsecRuntime.Config.OutOfBandOptions.RequestBodyInMemoryLimit != nil {
|
|
outbandCfg = outbandCfg.WithRequestBodyInMemoryLimit(*r.AppsecRuntime.Config.OutOfBandOptions.RequestBodyInMemoryLimit)
|
|
}
|
|
r.AppsecOutbandEngine, err = coraza.NewWAF(outbandCfg)
|
|
if err != nil {
|
|
return fmt.Errorf("unable to initialize outband engine : %w", err)
|
|
}
|
|
|
|
if r.AppsecRuntime.DisabledInBandRulesTags != nil {
|
|
for _, tag := range r.AppsecRuntime.DisabledInBandRulesTags {
|
|
r.AppsecInbandEngine.GetRuleGroup().DeleteByTag(tag)
|
|
}
|
|
}
|
|
|
|
if r.AppsecRuntime.DisabledOutOfBandRulesTags != nil {
|
|
for _, tag := range r.AppsecRuntime.DisabledOutOfBandRulesTags {
|
|
r.AppsecOutbandEngine.GetRuleGroup().DeleteByTag(tag)
|
|
}
|
|
}
|
|
|
|
if r.AppsecRuntime.DisabledInBandRuleIds != nil {
|
|
for _, id := range r.AppsecRuntime.DisabledInBandRuleIds {
|
|
r.AppsecInbandEngine.GetRuleGroup().DeleteByID(id)
|
|
}
|
|
}
|
|
|
|
if r.AppsecRuntime.DisabledOutOfBandRuleIds != nil {
|
|
for _, id := range r.AppsecRuntime.DisabledOutOfBandRuleIds {
|
|
r.AppsecOutbandEngine.GetRuleGroup().DeleteByID(id)
|
|
}
|
|
}
|
|
|
|
r.logger.Tracef("Loaded inband rules: %+v", r.AppsecInbandEngine.GetRuleGroup().GetRules())
|
|
r.logger.Tracef("Loaded outband rules: %+v", r.AppsecOutbandEngine.GetRuleGroup().GetRules())
|
|
|
|
return nil
|
|
}
|
|
|
|
func (r *AppsecRunner) processRequest(tx appsec.ExtendedTransaction, request *appsec.ParsedRequest) error {
|
|
var in *corazatypes.Interruption
|
|
var err error
|
|
|
|
if request.Tx.IsRuleEngineOff() {
|
|
r.logger.Debugf("rule engine is off, skipping")
|
|
return nil
|
|
}
|
|
|
|
defer func() {
|
|
request.Tx.ProcessLogging()
|
|
//We don't close the transaction here, as it will reset coraza internal state and break variable tracking
|
|
|
|
err := r.AppsecRuntime.ProcessPostEvalRules(request)
|
|
if err != nil {
|
|
r.logger.Errorf("unable to process PostEval rules: %s", err)
|
|
}
|
|
}()
|
|
|
|
//pre eval (expr) rules
|
|
err = r.AppsecRuntime.ProcessPreEvalRules(request)
|
|
if err != nil {
|
|
r.logger.Errorf("unable to process PreEval rules: %s", err)
|
|
//FIXME: should we abort here ?
|
|
}
|
|
|
|
request.Tx.ProcessConnection(request.ClientIP, 0, "", 0)
|
|
|
|
for k, v := range request.Args {
|
|
for _, vv := range v {
|
|
request.Tx.AddGetRequestArgument(k, vv)
|
|
}
|
|
}
|
|
|
|
request.Tx.ProcessURI(request.URI, request.Method, request.Proto)
|
|
|
|
for k, vr := range request.Headers {
|
|
for _, v := range vr {
|
|
request.Tx.AddRequestHeader(k, v)
|
|
}
|
|
}
|
|
|
|
if request.ClientHost != "" {
|
|
request.Tx.AddRequestHeader("Host", request.ClientHost)
|
|
request.Tx.SetServerName(request.ClientHost)
|
|
}
|
|
|
|
if request.TransferEncoding != nil {
|
|
request.Tx.AddRequestHeader("Transfer-Encoding", request.TransferEncoding[0])
|
|
}
|
|
|
|
in = request.Tx.ProcessRequestHeaders()
|
|
|
|
if in != nil {
|
|
r.logger.Infof("inband rules matched for headers : %s", in.Action)
|
|
return nil
|
|
}
|
|
|
|
if len(request.Body) > 0 {
|
|
in, _, err = request.Tx.WriteRequestBody(request.Body)
|
|
if err != nil {
|
|
r.logger.Errorf("unable to write request body : %s", err)
|
|
return err
|
|
}
|
|
if in != nil {
|
|
return nil
|
|
}
|
|
}
|
|
|
|
in, err = request.Tx.ProcessRequestBody()
|
|
if err != nil {
|
|
r.logger.Errorf("unable to process request body : %s", err)
|
|
return err
|
|
}
|
|
|
|
if in != nil {
|
|
r.logger.Debugf("rules matched for body : %d", in.RuleID)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (r *AppsecRunner) ProcessInBandRules(request *appsec.ParsedRequest) error {
|
|
tx := appsec.NewExtendedTransaction(r.AppsecInbandEngine, request.UUID)
|
|
r.AppsecRuntime.InBandTx = tx
|
|
request.Tx = tx
|
|
if len(r.AppsecRuntime.InBandRules) == 0 {
|
|
return nil
|
|
}
|
|
err := r.processRequest(tx, request)
|
|
return err
|
|
}
|
|
|
|
func (r *AppsecRunner) ProcessOutOfBandRules(request *appsec.ParsedRequest) error {
|
|
tx := appsec.NewExtendedTransaction(r.AppsecOutbandEngine, request.UUID)
|
|
r.AppsecRuntime.OutOfBandTx = tx
|
|
request.Tx = tx
|
|
if len(r.AppsecRuntime.OutOfBandRules) == 0 {
|
|
return nil
|
|
}
|
|
err := r.processRequest(tx, request)
|
|
return err
|
|
}
|
|
|
|
func (r *AppsecRunner) handleInBandInterrupt(request *appsec.ParsedRequest) {
|
|
//create the associated event for crowdsec itself
|
|
evt, err := EventFromRequest(request, r.Labels)
|
|
if err != nil {
|
|
//let's not interrupt the pipeline for this
|
|
r.logger.Errorf("unable to create event from request : %s", err)
|
|
}
|
|
err = r.AccumulateTxToEvent(&evt, request)
|
|
if err != nil {
|
|
r.logger.Errorf("unable to accumulate tx to event : %s", err)
|
|
}
|
|
if in := request.Tx.Interruption(); in != nil {
|
|
r.logger.Debugf("inband rules matched : %d", in.RuleID)
|
|
r.AppsecRuntime.Response.InBandInterrupt = true
|
|
r.AppsecRuntime.Response.BouncerHTTPResponseCode = r.AppsecRuntime.Config.BouncerBlockedHTTPCode
|
|
r.AppsecRuntime.Response.UserHTTPResponseCode = r.AppsecRuntime.Config.UserBlockedHTTPCode
|
|
r.AppsecRuntime.Response.Action = r.AppsecRuntime.DefaultRemediation
|
|
|
|
if _, ok := r.AppsecRuntime.RemediationById[in.RuleID]; ok {
|
|
r.AppsecRuntime.Response.Action = r.AppsecRuntime.RemediationById[in.RuleID]
|
|
}
|
|
|
|
for tag, remediation := range r.AppsecRuntime.RemediationByTag {
|
|
if slices.Contains[[]string, string](in.Tags, tag) {
|
|
r.AppsecRuntime.Response.Action = remediation
|
|
}
|
|
}
|
|
|
|
err = r.AppsecRuntime.ProcessOnMatchRules(request, evt)
|
|
if err != nil {
|
|
r.logger.Errorf("unable to process OnMatch rules: %s", err)
|
|
return
|
|
}
|
|
|
|
// Should the in band match trigger an overflow ?
|
|
if r.AppsecRuntime.Response.SendAlert {
|
|
appsecOvlfw, err := AppsecEventGeneration(evt, request.HTTPRequest)
|
|
if err != nil {
|
|
r.logger.Errorf("unable to generate appsec event : %s", err)
|
|
return
|
|
}
|
|
if appsecOvlfw != nil {
|
|
r.outChan <- *appsecOvlfw
|
|
}
|
|
}
|
|
|
|
// Should the in band match trigger an event ?
|
|
if r.AppsecRuntime.Response.SendEvent {
|
|
r.outChan <- evt
|
|
}
|
|
|
|
}
|
|
}
|
|
|
|
func (r *AppsecRunner) handleOutBandInterrupt(request *appsec.ParsedRequest) {
|
|
evt, err := EventFromRequest(request, r.Labels)
|
|
if err != nil {
|
|
//let's not interrupt the pipeline for this
|
|
r.logger.Errorf("unable to create event from request : %s", err)
|
|
}
|
|
err = r.AccumulateTxToEvent(&evt, request)
|
|
if err != nil {
|
|
r.logger.Errorf("unable to accumulate tx to event : %s", err)
|
|
}
|
|
if in := request.Tx.Interruption(); in != nil {
|
|
r.logger.Debugf("outband rules matched : %d", in.RuleID)
|
|
r.AppsecRuntime.Response.OutOfBandInterrupt = true
|
|
|
|
err = r.AppsecRuntime.ProcessOnMatchRules(request, evt)
|
|
if err != nil {
|
|
r.logger.Errorf("unable to process OnMatch rules: %s", err)
|
|
return
|
|
}
|
|
// Should the match trigger an event ?
|
|
if r.AppsecRuntime.Response.SendEvent {
|
|
r.outChan <- evt
|
|
}
|
|
|
|
// Should the match trigger an overflow ?
|
|
if r.AppsecRuntime.Response.SendAlert {
|
|
appsecOvlfw, err := AppsecEventGeneration(evt, request.HTTPRequest)
|
|
if err != nil {
|
|
r.logger.Errorf("unable to generate appsec event : %s", err)
|
|
return
|
|
}
|
|
r.outChan <- *appsecOvlfw
|
|
}
|
|
}
|
|
}
|
|
|
|
func (r *AppsecRunner) handleRequest(request *appsec.ParsedRequest) {
|
|
r.AppsecRuntime.Logger = r.AppsecRuntime.Logger.WithField("request_uuid", request.UUID)
|
|
logger := r.logger.WithField("request_uuid", request.UUID)
|
|
logger.Debug("Request received in runner")
|
|
r.AppsecRuntime.ClearResponse()
|
|
|
|
request.IsInBand = true
|
|
request.IsOutBand = false
|
|
|
|
//to measure the time spent in the Application Security Engine for InBand rules
|
|
startInBandParsing := time.Now()
|
|
startGlobalParsing := time.Now()
|
|
|
|
//inband appsec rules
|
|
err := r.ProcessInBandRules(request)
|
|
if err != nil {
|
|
logger.Errorf("unable to process InBand rules: %s", err)
|
|
return
|
|
}
|
|
|
|
// time spent to process in band rules
|
|
inBandParsingElapsed := time.Since(startInBandParsing)
|
|
AppsecInbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddrNormalized, "appsec_engine": request.AppsecEngine}).Observe(inBandParsingElapsed.Seconds())
|
|
|
|
if request.Tx.IsInterrupted() {
|
|
r.handleInBandInterrupt(request)
|
|
}
|
|
|
|
// send back the result to the HTTP handler for the InBand part
|
|
request.ResponseChannel <- r.AppsecRuntime.Response
|
|
|
|
//Now let's process the out of band rules
|
|
|
|
request.IsInBand = false
|
|
request.IsOutBand = true
|
|
r.AppsecRuntime.Response.SendAlert = false
|
|
r.AppsecRuntime.Response.SendEvent = true
|
|
|
|
//FIXME: This is a bit of a hack to avoid confusion with the transaction if we do not have any inband rules.
|
|
//We should probably have different transaction (or even different request object) for inband and out of band rules
|
|
if len(r.AppsecRuntime.OutOfBandRules) > 0 {
|
|
//to measure the time spent in the Application Security Engine for OutOfBand rules
|
|
startOutOfBandParsing := time.Now()
|
|
|
|
err = r.ProcessOutOfBandRules(request)
|
|
if err != nil {
|
|
logger.Errorf("unable to process OutOfBand rules: %s", err)
|
|
return
|
|
}
|
|
|
|
// time spent to process out of band rules
|
|
outOfBandParsingElapsed := time.Since(startOutOfBandParsing)
|
|
AppsecOutbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddrNormalized, "appsec_engine": request.AppsecEngine}).Observe(outOfBandParsingElapsed.Seconds())
|
|
if request.Tx.IsInterrupted() {
|
|
r.handleOutBandInterrupt(request)
|
|
}
|
|
}
|
|
// time spent to process inband AND out of band rules
|
|
globalParsingElapsed := time.Since(startGlobalParsing)
|
|
AppsecGlobalParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddrNormalized, "appsec_engine": request.AppsecEngine}).Observe(globalParsingElapsed.Seconds())
|
|
}
|
|
|
|
func (r *AppsecRunner) Run(t *tomb.Tomb) error {
|
|
r.logger.Infof("Appsec Runner ready to process event")
|
|
for {
|
|
select {
|
|
case <-t.Dying():
|
|
r.logger.Infof("Appsec Runner is dying")
|
|
return nil
|
|
case request := <-r.inChan:
|
|
r.handleRequest(&request)
|
|
}
|
|
}
|
|
}
|