mirrors/crowdsec
1
0
Fork 0
mirror of https://github.com/crowdsecurity/crowdsec.git synced 2025-05-10 20:05:55 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

refact parser Init: argument types (#3578)

Browse source 
* refact parser Init: argument types
* lint
* tests
* rename struct field; drop redundant nil check
This commit is contained in:
mmetc 2025-04-23 21:30:03 +02:00 • committed by GitHub
parent 7c4e91d304
commit d46cef1bcb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 27 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

2
cmd/crowdsec/crowdsec.go
View file

@ -144,7 +144,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.H
outputsTomb.Go(func() error {
defer trace.CatchPanic("crowdsec/runOutput")
return runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, apiClient)
return runOutput(inputEventChan, outputEventChan, buckets, *parsers.PovfwCtx, parsers.Povfwnodes, apiClient)
})
}

5
pkg/parser/node_test.go
View file

@ -7,7 +7,7 @@ import (
)
func TestParserConfigs(t *testing.T) {
pctx, err := Init(map[string]interface{}{"patterns": "../../config/patterns/", "data": "./tests/"})
pctx, err := NewUnixParserCtx("../../config/patterns/", "./tests/")
if err != nil {
t.Fatalf("unable to load patterns : %s", err)
}
@ -47,11 +47,13 @@ func TestParserConfigs(t *testing.T) {
{Key: string("MYGROKBIS"), Value: string("[a-z]")},
}, Grok: GrokPattern{RegexpValue: "^x%{MYGROKBIS:extr}$", TargetField: "t"}}, false, true},
}
for idx := range CfgTests {
err := CfgTests[idx].NodeCfg.compile(pctx, EnricherCtx{})
if CfgTests[idx].Compiles && err != nil {
t.Fatalf("Compile: (%d/%d) expected valid, got : %s", idx+1, len(CfgTests), err)
}
if !CfgTests[idx].Compiles && err == nil {
t.Fatalf("Compile: (%d/%d) expected error", idx+1, len(CfgTests))
}
@ -60,6 +62,7 @@ func TestParserConfigs(t *testing.T) {
if CfgTests[idx].Valid && err != nil {
t.Fatalf("Valid: (%d/%d) expected valid, got : %s", idx+1, len(CfgTests), err)
}
if !CfgTests[idx].Valid && err == nil {
t.Fatalf("Valid: (%d/%d) expected error", idx+1, len(CfgTests))
}

5
pkg/parser/parsing_test.go
View file

@ -7,6 +7,7 @@ import (
"html/template"
"io"
"os"
"path/filepath"
"sort"
"strings"
"testing"
@ -179,7 +180,7 @@ func prepTests(t require.TestingT) (*UnixParserCtx, EnricherCtx) {
/* this should be refactored to 2 lines :p */
// Init the parser
pctx, err = Init(map[string]interface{}{"patterns": cfgdir + string("/patterns/"), "data": "./tests/"})
pctx, err = NewUnixParserCtx(filepath.Join(cfgdir, "patterns"), "./tests/")
require.NoError(t, err, "parser init failed")
return pctx, ectx
@ -403,7 +404,7 @@ func TestGeneratePatternsDoc(t *testing.T) {
return
}
pctx, err := Init(map[string]interface{}{"patterns": "../../config/patterns/", "data": "./tests/"})
pctx, err := NewUnixParserCtx("../../config/patterns/", "./tests/")
require.NoError(t, err, "unable to load patterns")
log.Infof("-> %s", spew.Sdump(pctx))

36
pkg/parser/unix_parser.go
View file

@ -25,7 +25,7 @@ type UnixParserCtx struct {
type Parsers struct {
Ctx *UnixParserCtx
Povfwctx *UnixParserCtx
PovfwCtx *UnixParserCtx
StageFiles []Stagefile
PovfwStageFiles []Stagefile
Nodes []Node
@ -33,24 +33,24 @@ type Parsers struct {
EnricherCtx EnricherCtx
}
func Init(c map[string]any) (*UnixParserCtx, error) {
func NewUnixParserCtx(patternDir string, dataDir string) (*UnixParserCtx, error) {
r := UnixParserCtx{}
r.Grok = grokky.NewBase()
r.Grok.UseRe2 = fflag.Re2GrokSupport.IsEnabled()
files, err := os.ReadDir(c["patterns"].(string))
files, err := os.ReadDir(patternDir)
if err != nil {
return nil, err
}
r.DataFolder = c["data"].(string)
r.DataFolder = dataDir
for _, file := range files {
if strings.Contains(file.Name(), ".") || file.IsDir() {
continue
}
if err := r.Grok.AddFromFile(filepath.Join(c["patterns"].(string), file.Name())); err != nil {
if err := r.Grok.AddFromFile(filepath.Join(patternDir, file.Name())); err != nil {
log.Errorf("failed to load pattern %s: %v", file.Name(), err)
return nil, err
}
@ -66,7 +66,7 @@ func Init(c map[string]any) (*UnixParserCtx, error) {
func NewParsers(hub *cwhub.Hub) *Parsers {
parsers := &Parsers{
Ctx: &UnixParserCtx{},
Povfwctx: &UnixParserCtx{},
PovfwCtx: &UnixParserCtx{},
StageFiles: make([]Stagefile, 0),
PovfwStageFiles: make([]Stagefile, 0),
}
@ -88,17 +88,13 @@ func NewParsers(hub *cwhub.Hub) *Parsers {
}
}
if parsers.StageFiles != nil {
sort.Slice(parsers.StageFiles, func(i, j int) bool {
return parsers.StageFiles[i].Filename < parsers.StageFiles[j].Filename
})
}
if parsers.PovfwStageFiles != nil {
sort.Slice(parsers.PovfwStageFiles, func(i, j int) bool {
return parsers.PovfwStageFiles[i].Filename < parsers.PovfwStageFiles[j].Filename
})
}
return parsers
}
@ -106,22 +102,16 @@ func NewParsers(hub *cwhub.Hub) *Parsers {
func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
var err error
patternsDir := cConfig.ConfigPaths.PatternDir
log.Infof("Loading grok library %s", patternsDir)
patternDir := cConfig.ConfigPaths.PatternDir
log.Infof("Loading grok library %s", patternDir)
/* load base regexps for two grok parsers */
parsers.Ctx, err = Init(map[string]any{
"patterns": patternsDir,
"data": cConfig.ConfigPaths.DataDir,
})
parsers.Ctx, err = NewUnixParserCtx(patternDir, cConfig.ConfigPaths.DataDir)
if err != nil {
return parsers, fmt.Errorf("failed to load parser patterns: %w", err)
}
parsers.Povfwctx, err = Init(map[string]any{
"patterns": patternsDir,
"data": cConfig.ConfigPaths.DataDir,
})
parsers.PovfwCtx, err = NewUnixParserCtx(patternDir, cConfig.ConfigPaths.DataDir)
if err != nil {
return parsers, fmt.Errorf("failed to load postovflw parser patterns: %w", err)
}
@ -150,7 +140,7 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
if len(parsers.PovfwStageFiles) > 0 {
log.Info("Loading postoverflow parsers")
parsers.Povfwnodes, err = LoadStages(parsers.PovfwStageFiles, parsers.Povfwctx, parsers.EnricherCtx)
parsers.Povfwnodes, err = LoadStages(parsers.PovfwStageFiles, parsers.PovfwCtx, parsers.EnricherCtx)
if err != nil {
return parsers, fmt.Errorf("failed to load postoverflow config: %w", err)
}
@ -162,13 +152,13 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
if cConfig.Prometheus != nil && cConfig.Prometheus.Enabled {
parsers.Ctx.Profiling = true
parsers.Povfwctx.Profiling = true
parsers.PovfwCtx.Profiling = true
}
/*
Reset CTX grok to reduce memory footprint after we compile all the patterns
*/
parsers.Ctx.Grok = grokky.Host{}
parsers.Povfwctx.Grok = grokky.Host{}
parsers.PovfwCtx.Grok = grokky.Host{}
parsers.StageFiles = []Stagefile{}
parsers.PovfwStageFiles = []Stagefile{}