mirrors/crowdsec
1
0
Fork 0
mirror of https://github.com/crowdsecurity/crowdsec.git synced 2025-05-11 04:15:54 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

refact parser Init: argument types (#3578)

Browse source 
* refact parser Init: argument types
* lint
* tests
* rename struct field; drop redundant nil check
This commit is contained in:
mmetc 2025-04-23 21:30:03 +02:00 • committed by GitHub
parent 7c4e91d304
commit d46cef1bcb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 27 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

2
cmd/crowdsec/crowdsec.go
View file

@ -144,7 +144,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.H
outputsTomb.Go(func() error { outputsTomb.Go(func() error {
defer trace.CatchPanic("crowdsec/runOutput") defer trace.CatchPanic("crowdsec/runOutput")
return runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, apiClient) return runOutput(inputEventChan, outputEventChan, buckets, *parsers.PovfwCtx, parsers.Povfwnodes, apiClient)
}) })
} }

5
pkg/parser/node_test.go
View file

@ -7,7 +7,7 @@ import (
) )
func TestParserConfigs(t *testing.T) { func TestParserConfigs(t *testing.T) {
pctx, err := Init(map[string]interface{}{"patterns": "../../config/patterns/", "data": "./tests/"}) pctx, err := NewUnixParserCtx("../../config/patterns/", "./tests/")
if err != nil { if err != nil {
t.Fatalf("unable to load patterns : %s", err) t.Fatalf("unable to load patterns : %s", err)
} }
@ -47,11 +47,13 @@ func TestParserConfigs(t *testing.T) {
{Key: string("MYGROKBIS"), Value: string("[a-z]")}, {Key: string("MYGROKBIS"), Value: string("[a-z]")},
}, Grok: GrokPattern{RegexpValue: "^x%{MYGROKBIS:extr}$", TargetField: "t"}}, false, true}, }, Grok: GrokPattern{RegexpValue: "^x%{MYGROKBIS:extr}$", TargetField: "t"}}, false, true},
} }
for idx := range CfgTests { for idx := range CfgTests {
err := CfgTests[idx].NodeCfg.compile(pctx, EnricherCtx{}) err := CfgTests[idx].NodeCfg.compile(pctx, EnricherCtx{})
if CfgTests[idx].Compiles && err != nil { if CfgTests[idx].Compiles && err != nil {
t.Fatalf("Compile: (%d/%d) expected valid, got : %s", idx+1, len(CfgTests), err) t.Fatalf("Compile: (%d/%d) expected valid, got : %s", idx+1, len(CfgTests), err)
} }
if !CfgTests[idx].Compiles && err == nil { if !CfgTests[idx].Compiles && err == nil {
t.Fatalf("Compile: (%d/%d) expected error", idx+1, len(CfgTests)) t.Fatalf("Compile: (%d/%d) expected error", idx+1, len(CfgTests))
} }
@ -60,6 +62,7 @@ func TestParserConfigs(t *testing.T) {
if CfgTests[idx].Valid && err != nil { if CfgTests[idx].Valid && err != nil {
t.Fatalf("Valid: (%d/%d) expected valid, got : %s", idx+1, len(CfgTests), err) t.Fatalf("Valid: (%d/%d) expected valid, got : %s", idx+1, len(CfgTests), err)
} }
if !CfgTests[idx].Valid && err == nil { if !CfgTests[idx].Valid && err == nil {
t.Fatalf("Valid: (%d/%d) expected error", idx+1, len(CfgTests)) t.Fatalf("Valid: (%d/%d) expected error", idx+1, len(CfgTests))
} }

5
pkg/parser/parsing_test.go
View file

@ -7,6 +7,7 @@ import (
"html/template" "html/template"
"io" "io"
"os" "os"
"path/filepath"
"sort" "sort"
"strings" "strings"
"testing" "testing"
@ -179,7 +180,7 @@ func prepTests(t require.TestingT) (*UnixParserCtx, EnricherCtx) {
/* this should be refactored to 2 lines :p */ /* this should be refactored to 2 lines :p */
// Init the parser // Init the parser
pctx, err = Init(map[string]interface{}{"patterns": cfgdir + string("/patterns/"), "data": "./tests/"}) pctx, err = NewUnixParserCtx(filepath.Join(cfgdir, "patterns"), "./tests/")
require.NoError(t, err, "parser init failed") require.NoError(t, err, "parser init failed")
return pctx, ectx return pctx, ectx
@ -403,7 +404,7 @@ func TestGeneratePatternsDoc(t *testing.T) {
return return
} }
pctx, err := Init(map[string]interface{}{"patterns": "../../config/patterns/", "data": "./tests/"}) pctx, err := NewUnixParserCtx("../../config/patterns/", "./tests/")
require.NoError(t, err, "unable to load patterns") require.NoError(t, err, "unable to load patterns")
log.Infof("-> %s", spew.Sdump(pctx)) log.Infof("-> %s", spew.Sdump(pctx))

36
pkg/parser/unix_parser.go
View file

@ -25,7 +25,7 @@ type UnixParserCtx struct {
type Parsers struct { type Parsers struct {
Ctx *UnixParserCtx Ctx *UnixParserCtx
Povfwctx *UnixParserCtx PovfwCtx *UnixParserCtx
StageFiles []Stagefile StageFiles []Stagefile
PovfwStageFiles []Stagefile PovfwStageFiles []Stagefile
Nodes []Node Nodes []Node
@ -33,24 +33,24 @@ type Parsers struct {
EnricherCtx EnricherCtx EnricherCtx EnricherCtx
} }
func Init(c map[string]any) (*UnixParserCtx, error) { func NewUnixParserCtx(patternDir string, dataDir string) (*UnixParserCtx, error) {
r := UnixParserCtx{} r := UnixParserCtx{}
r.Grok = grokky.NewBase() r.Grok = grokky.NewBase()
r.Grok.UseRe2 = fflag.Re2GrokSupport.IsEnabled() r.Grok.UseRe2 = fflag.Re2GrokSupport.IsEnabled()
files, err := os.ReadDir(c["patterns"].(string)) files, err := os.ReadDir(patternDir)
if err != nil { if err != nil {
return nil, err return nil, err
} }
r.DataFolder = c["data"].(string) r.DataFolder = dataDir
for _, file := range files { for _, file := range files {
if strings.Contains(file.Name(), ".") || file.IsDir() { if strings.Contains(file.Name(), ".") || file.IsDir() {
continue continue
} }
if err := r.Grok.AddFromFile(filepath.Join(c["patterns"].(string), file.Name())); err != nil { if err := r.Grok.AddFromFile(filepath.Join(patternDir, file.Name())); err != nil {
log.Errorf("failed to load pattern %s: %v", file.Name(), err) log.Errorf("failed to load pattern %s: %v", file.Name(), err)
return nil, err return nil, err
} }
@ -66,7 +66,7 @@ func Init(c map[string]any) (*UnixParserCtx, error) {
func NewParsers(hub *cwhub.Hub) *Parsers { func NewParsers(hub *cwhub.Hub) *Parsers {
parsers := &Parsers{ parsers := &Parsers{
Ctx: &UnixParserCtx{}, Ctx: &UnixParserCtx{},
Povfwctx: &UnixParserCtx{}, PovfwCtx: &UnixParserCtx{},
StageFiles: make([]Stagefile, 0), StageFiles: make([]Stagefile, 0),
PovfwStageFiles: make([]Stagefile, 0), PovfwStageFiles: make([]Stagefile, 0),
} }
@ -88,17 +88,13 @@ func NewParsers(hub *cwhub.Hub) *Parsers {
} }
} }
if parsers.StageFiles != nil {
sort.Slice(parsers.StageFiles, func(i, j int) bool { sort.Slice(parsers.StageFiles, func(i, j int) bool {
return parsers.StageFiles[i].Filename < parsers.StageFiles[j].Filename return parsers.StageFiles[i].Filename < parsers.StageFiles[j].Filename
}) })
}
if parsers.PovfwStageFiles != nil {
sort.Slice(parsers.PovfwStageFiles, func(i, j int) bool { sort.Slice(parsers.PovfwStageFiles, func(i, j int) bool {
return parsers.PovfwStageFiles[i].Filename < parsers.PovfwStageFiles[j].Filename return parsers.PovfwStageFiles[i].Filename < parsers.PovfwStageFiles[j].Filename
}) })
}
return parsers return parsers
} }
@ -106,22 +102,16 @@ func NewParsers(hub *cwhub.Hub) *Parsers {
func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) { func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
var err error var err error
patternsDir := cConfig.ConfigPaths.PatternDir patternDir := cConfig.ConfigPaths.PatternDir
log.Infof("Loading grok library %s", patternsDir) log.Infof("Loading grok library %s", patternDir)
/* load base regexps for two grok parsers */ /* load base regexps for two grok parsers */
parsers.Ctx, err = Init(map[string]any{ parsers.Ctx, err = NewUnixParserCtx(patternDir, cConfig.ConfigPaths.DataDir)
"patterns": patternsDir,
"data": cConfig.ConfigPaths.DataDir,
})
if err != nil { if err != nil {
return parsers, fmt.Errorf("failed to load parser patterns: %w", err) return parsers, fmt.Errorf("failed to load parser patterns: %w", err)
} }
parsers.Povfwctx, err = Init(map[string]any{ parsers.PovfwCtx, err = NewUnixParserCtx(patternDir, cConfig.ConfigPaths.DataDir)
"patterns": patternsDir,
"data": cConfig.ConfigPaths.DataDir,
})
if err != nil { if err != nil {
return parsers, fmt.Errorf("failed to load postovflw parser patterns: %w", err) return parsers, fmt.Errorf("failed to load postovflw parser patterns: %w", err)
} }
@ -150,7 +140,7 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
if len(parsers.PovfwStageFiles) > 0 { if len(parsers.PovfwStageFiles) > 0 {
log.Info("Loading postoverflow parsers") log.Info("Loading postoverflow parsers")
parsers.Povfwnodes, err = LoadStages(parsers.PovfwStageFiles, parsers.Povfwctx, parsers.EnricherCtx) parsers.Povfwnodes, err = LoadStages(parsers.PovfwStageFiles, parsers.PovfwCtx, parsers.EnricherCtx)
if err != nil { if err != nil {
return parsers, fmt.Errorf("failed to load postoverflow config: %w", err) return parsers, fmt.Errorf("failed to load postoverflow config: %w", err)
} }
@ -162,13 +152,13 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
if cConfig.Prometheus != nil && cConfig.Prometheus.Enabled { if cConfig.Prometheus != nil && cConfig.Prometheus.Enabled {
parsers.Ctx.Profiling = true parsers.Ctx.Profiling = true
parsers.Povfwctx.Profiling = true parsers.PovfwCtx.Profiling = true
} }
/* /*
Reset CTX grok to reduce memory footprint after we compile all the patterns Reset CTX grok to reduce memory footprint after we compile all the patterns
*/ */
parsers.Ctx.Grok = grokky.Host{} parsers.Ctx.Grok = grokky.Host{}
parsers.Povfwctx.Grok = grokky.Host{} parsers.PovfwCtx.Grok = grokky.Host{}
parsers.StageFiles = []Stagefile{} parsers.StageFiles = []Stagefile{}
parsers.PovfwStageFiles = []Stagefile{} parsers.PovfwStageFiles = []Stagefile{}