mirrors/crowdsec
1
0
Fork 0
mirror of https://github.com/crowdsecurity/crowdsec.git synced 2025-05-11 12:25:53 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Parallel hubtest (#3509)

Browse source 
Hubtests are now much faster and have a --max-jobs option which defaults to the number of cpu cores.
This commit is contained in:
mmetc 2025-03-17 11:27:09 +01:00 • committed by GitHub
parent f5400482a6
commit cab99643d1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 270 additions and 271 deletions
Download patch file Download diff file Expand all files Collapse all files

12
.github/workflows/bats-hub.yml vendored
View file

@ -10,9 +10,6 @@ on:
jobs: jobs:
build: build:
strategy:
matrix:
test-file: ["hub-1.bats", "hub-2.bats", "hub-3.bats"]
name: "Functional tests" name: "Functional tests"
runs-on: ubuntu-latest runs-on: ubuntu-latest
@ -46,11 +43,14 @@ jobs:
- name: "Run hub tests" - name: "Run hub tests"
run: | run: |
./test/bin/generate-hub-tests PATH=$(pwd)/test/local/bin:$PATH
./test/run-tests ./test/dyn-bats/${{ matrix.test-file }} --formatter $(pwd)/test/lib/color-formatter ./test/instance-data load
git clone --depth 1 https://github.com/crowdsecurity/hub.git ./hub
cd ./hub
cscli hubtest run --all --clean --max-jobs 8
- name: "Collect hub coverage" - name: "Collect hub coverage"
run: ./test/bin/collect-hub-coverage >> $GITHUB_ENV run: ./test/bin/collect-hub-coverage ./hub >> $GITHUB_ENV
- name: "Create Parsers badge" - name: "Create Parsers badge"
uses: schneegans/dynamic-badges-action@v1.7.0 uses: schneegans/dynamic-badges-action@v1.7.0

38
cmd/crowdsec-cli/clihubtest/clean.go
View file

@ -1,33 +1,55 @@
package clihubtest package clihubtest
import ( import (
"errors"
"fmt" "fmt"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/args" "github.com/crowdsecurity/crowdsec/pkg/hubtest"
) )
func (cli *cliHubTest) newCleanCmd() *cobra.Command { func (cli *cliHubTest) newCleanCmd() *cobra.Command {
var all bool
cmd := &cobra.Command{ cmd := &cobra.Command{
Use: "clean", Use: "clean",
Short: "clean [test_name]", Short: "clean [test_name]",
Args: args.MinimumNArgs(1),
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error { RunE: func(_ *cobra.Command, args []string) error {
for _, testName := range args { if !all && len(args) == 0 {
test, err := hubPtr.LoadTestItem(testName) return errors.New("please provide test to run or --all flag")
if err != nil { }
return fmt.Errorf("unable to load test '%s': %w", testName, err)
fmt.Println("Cleaning test data...")
tests := []*hubtest.HubTestItem{}
if all {
if err := hubPtr.LoadAllTests(); err != nil {
return fmt.Errorf("unable to load all tests: %w", err)
} }
if err := test.Clean(); err != nil {
return fmt.Errorf("unable to clean test '%s' env: %w", test.Name, err) tests = hubPtr.Tests
} else {
for _, testName := range args {
test, err := hubPtr.LoadTestItem(testName)
if err != nil {
return fmt.Errorf("unable to load test '%s': %w", testName, err)
}
tests = append(tests, test)
} }
} }
for _, test := range tests {
test.Clean()
}
return nil return nil
}, },
} }
cmd.Flags().BoolVar(&all, "all", false, "Run all tests")
return cmd return cmd
} }

12
cmd/crowdsec-cli/clihubtest/coverage.go
View file

@ -14,12 +14,12 @@ import (
) )
// getCoverage returns the coverage and the percentage of tests that passed // getCoverage returns the coverage and the percentage of tests that passed
func getCoverage(show bool, getCoverageFunc func() ([]hubtest.Coverage, error)) ([]hubtest.Coverage, int, error) { func getCoverage(show bool, getCoverageFunc func(string) ([]hubtest.Coverage, error), hubDir string) ([]hubtest.Coverage, int, error) {
if !show { if !show {
return nil, 0, nil return nil, 0, nil
} }
coverage, err := getCoverageFunc() coverage, err := getCoverageFunc(hubDir)
if err != nil { if err != nil {
return nil, 0, fmt.Errorf("while getting coverage: %w", err) return nil, 0, fmt.Errorf("while getting coverage: %w", err)
} }
@ -46,7 +46,7 @@ func (cli *cliHubTest) coverage(showScenarioCov bool, showParserCov bool, showAp
// for this one we explicitly don't do for appsec // for this one we explicitly don't do for appsec
if err := HubTest.LoadAllTests(); err != nil { if err := HubTest.LoadAllTests(); err != nil {
return fmt.Errorf("unable to load all tests: %+v", err) return fmt.Errorf("unable to load all tests: %w", err)
} }
var err error var err error
@ -58,17 +58,17 @@ func (cli *cliHubTest) coverage(showScenarioCov bool, showParserCov bool, showAp
showAppsecCov = true showAppsecCov = true
} }
parserCoverage, parserCoveragePercent, err := getCoverage(showParserCov, HubTest.GetParsersCoverage) parserCoverage, parserCoveragePercent, err := getCoverage(showParserCov, HubTest.GetParsersCoverage, cfg.Hub.HubDir)
if err != nil { if err != nil {
return err return err
} }
scenarioCoverage, scenarioCoveragePercent, err := getCoverage(showScenarioCov, HubTest.GetScenariosCoverage) scenarioCoverage, scenarioCoveragePercent, err := getCoverage(showScenarioCov, HubTest.GetScenariosCoverage, cfg.Hub.HubDir)
if err != nil { if err != nil {
return err return err
} }
appsecRuleCoverage, appsecRuleCoveragePercent, err := getCoverage(showAppsecCov, HubTest.GetAppsecCoverage) appsecRuleCoverage, appsecRuleCoveragePercent, err := getCoverage(showAppsecCov, HubTest.GetAppsecCoverage, cfg.Hub.HubDir)
if err != nil { if err != nil {
return err return err
} }

21
cmd/crowdsec-cli/clihubtest/explain.go
View file

@ -1,19 +1,19 @@
package clihubtest package clihubtest
import ( import (
"context"
"fmt" "fmt"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/dumps"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/args" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/args"
"github.com/crowdsecurity/crowdsec/pkg/dumps"
) )
func (cli *cliHubTest) explain(testName string, details bool, skipOk bool) error { func (cli *cliHubTest) explain(ctx context.Context, testName string, details bool, skipOk bool) error {
test, err := HubTest.LoadTestItem(testName) test, err := HubTest.LoadTestItem(testName)
if err != nil { if err != nil {
return fmt.Errorf("can't load test: %+v", err) return fmt.Errorf("can't load test: %w", err)
} }
cfg := cli.cfg() cfg := cli.cfg()
@ -21,8 +21,8 @@ func (cli *cliHubTest) explain(testName string, details bool, skipOk bool) error
err = test.ParserAssert.LoadTest(test.ParserResultFile) err = test.ParserAssert.LoadTest(test.ParserResultFile)
if err != nil { if err != nil {
if err = test.Run(patternDir); err != nil { if err = test.Run(ctx, patternDir); err != nil {
return fmt.Errorf("running test '%s' failed: %+v", test.Name, err) return fmt.Errorf("running test '%s' failed: %w", test.Name, err)
} }
if err = test.ParserAssert.LoadTest(test.ParserResultFile); err != nil { if err = test.ParserAssert.LoadTest(test.ParserResultFile); err != nil {
@ -32,8 +32,8 @@ func (cli *cliHubTest) explain(testName string, details bool, skipOk bool) error
err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile) err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
if err != nil { if err != nil {
if err = test.Run(patternDir); err != nil { if err = test.Run(ctx, patternDir); err != nil {
return fmt.Errorf("running test '%s' failed: %+v", test.Name, err) return fmt.Errorf("running test '%s' failed: %w", test.Name, err)
} }
if err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile); err != nil { if err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile); err != nil {
@ -62,9 +62,10 @@ func (cli *cliHubTest) newExplainCmd() *cobra.Command {
Short: "explain [test_name]", Short: "explain [test_name]",
Args: args.MinimumNArgs(1), Args: args.MinimumNArgs(1),
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
for _, testName := range args { for _, testName := range args {
if err := cli.explain(testName, details, skipOk); err != nil { if err := cli.explain(ctx, testName, details, skipOk); err != nil {
return err return err
} }
} }

80
cmd/crowdsec-cli/clihubtest/run.go
View file

@ -1,34 +1,36 @@
package clihubtest package clihubtest
import ( import (
"context"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"os" "runtime"
"strings" "strings"
"github.com/AlecAivazis/survey/v2" "github.com/AlecAivazis/survey/v2"
"github.com/fatih/color" "github.com/fatih/color"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"golang.org/x/sync/errgroup"
"github.com/crowdsecurity/crowdsec/pkg/emoji" "github.com/crowdsecurity/crowdsec/pkg/emoji"
"github.com/crowdsecurity/crowdsec/pkg/hubtest" "github.com/crowdsecurity/crowdsec/pkg/hubtest"
) )
func (cli *cliHubTest) run(runAll bool, nucleiTargetHost string, appSecHost string, args []string) error { func (cli *cliHubTest) run(ctx context.Context, all bool, nucleiTargetHost string, appSecHost string, args []string, maxJobs uint) error {
cfg := cli.cfg() cfg := cli.cfg()
if !runAll && len(args) == 0 { if !all && len(args) == 0 {
return errors.New("please provide test to run or --all flag") return errors.New("please provide test to run or --all flag")
} }
hubPtr.NucleiTargetHost = nucleiTargetHost hubPtr.NucleiTargetHost = nucleiTargetHost
hubPtr.AppSecHost = appSecHost hubPtr.AppSecHost = appSecHost
if runAll { if all {
if err := hubPtr.LoadAllTests(); err != nil { if err := hubPtr.LoadAllTests(); err != nil {
return fmt.Errorf("unable to load all tests: %+v", err) return fmt.Errorf("unable to load all tests: %w", err)
} }
} else { } else {
for _, testName := range args { for _, testName := range args {
@ -39,23 +41,23 @@ func (cli *cliHubTest) run(runAll bool, nucleiTargetHost string, appSecHost stri
} }
} }
// set timezone to avoid DST issues
os.Setenv("TZ", "UTC")
patternDir := cfg.ConfigPaths.PatternDir patternDir := cfg.ConfigPaths.PatternDir
var eg errgroup.Group
eg.SetLimit(int(maxJobs))
for _, test := range hubPtr.Tests { for _, test := range hubPtr.Tests {
if cfg.Cscli.Output == "human" { if cfg.Cscli.Output == "human" {
log.Infof("Running test '%s'", test.Name) fmt.Printf("Running test '%s'\n", test.Name)
} }
err := test.Run(patternDir) eg.Go(func() error {
if err != nil { return test.Run(ctx, patternDir)
log.Errorf("running test '%s' failed: %+v", test.Name, err) })
}
} }
return nil return eg.Wait()
} }
func printParserFailures(test *hubtest.HubTestItem) { func printParserFailures(test *hubtest.HubTestItem) {
@ -101,24 +103,31 @@ func printScenarioFailures(test *hubtest.HubTestItem) {
func (cli *cliHubTest) newRunCmd() *cobra.Command { func (cli *cliHubTest) newRunCmd() *cobra.Command {
var ( var (
noClean bool noClean bool
runAll bool all bool
reportSuccess bool
forceClean bool forceClean bool
nucleiTargetHost string nucleiTargetHost string
appSecHost string appSecHost string
) )
maxJobs := uint(runtime.NumCPU())
cmd := &cobra.Command{ cmd := &cobra.Command{
Use: "run", Use: "run",
Short: "run [test_name]", Short: "run [test_name]",
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return cli.run(runAll, nucleiTargetHost, appSecHost, args) if all {
fmt.Printf("Running all tests (max_jobs: %d)\n", maxJobs)
}
return cli.run(cmd.Context(), all, nucleiTargetHost, appSecHost, args, maxJobs)
}, },
PersistentPostRunE: func(_ *cobra.Command, _ []string) error { PersistentPostRunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg() cfg := cli.cfg()
success := true success := true
testResult := make(map[string]bool) testMap := make(map[string]*hubtest.HubTestItem)
for _, test := range hubPtr.Tests { for _, test := range hubPtr.Tests {
if test.AutoGen && !isAppsecTest { if test.AutoGen && !isAppsecTest {
if test.ParserAssert.AutoGenAssert { if test.ParserAssert.AutoGenAssert {
@ -132,22 +141,15 @@ func (cli *cliHubTest) newRunCmd() *cobra.Command {
fmt.Println(test.ScenarioAssert.AutoGenAssertData) fmt.Println(test.ScenarioAssert.AutoGenAssertData)
} }
if !noClean { if !noClean {
if err := test.Clean(); err != nil { test.Clean()
return fmt.Errorf("unable to clean test '%s' env: %w", test.Name, err)
}
} }
return fmt.Errorf("please fill your assert file(s) for test '%s', exiting", test.Name) return fmt.Errorf("please fill your assert file(s) for test '%s', exiting", test.Name)
} }
testResult[test.Name] = test.Success testMap[test.Name] = test
if test.Success { if test.Success {
if cfg.Cscli.Output == "human" {
log.Infof("Test '%s' passed successfully (%d assertions)\n", test.Name, test.ParserAssert.NbAssert+test.ScenarioAssert.NbAssert)
}
if !noClean { if !noClean {
if err := test.Clean(); err != nil { test.Clean()
return fmt.Errorf("unable to clean test '%s' env: %w", test.Name, err)
}
} }
} else { } else {
success = false success = false
@ -157,7 +159,7 @@ func (cli *cliHubTest) newRunCmd() *cobra.Command {
printScenarioFailures(test) printScenarioFailures(test)
if !forceClean && !noClean { if !forceClean && !noClean {
prompt := &survey.Confirm{ prompt := &survey.Confirm{
Message: fmt.Sprintf("\nDo you want to remove runtime folder for test '%s'? (default: Yes)", test.Name), Message: fmt.Sprintf("Do you want to remove runtime and result folder for '%s'?", test.Name),
Default: true, Default: true,
} }
if err := survey.AskOne(prompt, &cleanTestEnv); err != nil { if err := survey.AskOne(prompt, &cleanTestEnv); err != nil {
@ -167,22 +169,20 @@ func (cli *cliHubTest) newRunCmd() *cobra.Command {
} }
if cleanTestEnv || forceClean { if cleanTestEnv || forceClean {
if err := test.Clean(); err != nil { test.Clean()
return fmt.Errorf("unable to clean test '%s' env: %w", test.Name, err)
}
} }
} }
} }
switch cfg.Cscli.Output { switch cfg.Cscli.Output {
case "human": case "human":
hubTestResultTable(color.Output, cfg.Cscli.Color, testResult) hubTestResultTable(color.Output, cfg.Cscli.Color, testMap, reportSuccess)
case "json": case "json":
jsonResult := make(map[string][]string, 0) jsonResult := make(map[string][]string, 0)
jsonResult["success"] = make([]string, 0) jsonResult["success"] = make([]string, 0)
jsonResult["fail"] = make([]string, 0) jsonResult["fail"] = make([]string, 0)
for testName, success := range testResult { for testName, test := range testMap {
if success { if test.Success {
jsonResult["success"] = append(jsonResult["success"], testName) jsonResult["success"] = append(jsonResult["success"], testName)
} else { } else {
jsonResult["fail"] = append(jsonResult["fail"], testName) jsonResult["fail"] = append(jsonResult["fail"], testName)
@ -198,7 +198,11 @@ func (cli *cliHubTest) newRunCmd() *cobra.Command {
} }
if !success { if !success {
return errors.New("some tests failed") if reportSuccess {
return errors.New("some tests failed")
}
return errors.New("some tests failed, use --report-success to show them all")
} }
return nil return nil
@ -209,7 +213,9 @@ func (cli *cliHubTest) newRunCmd() *cobra.Command {
cmd.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail") cmd.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail")
cmd.Flags().StringVar(&nucleiTargetHost, "target", hubtest.DefaultNucleiTarget, "Target for AppSec Test") cmd.Flags().StringVar(&nucleiTargetHost, "target", hubtest.DefaultNucleiTarget, "Target for AppSec Test")
cmd.Flags().StringVar(&appSecHost, "host", hubtest.DefaultAppsecHost, "Address to expose AppSec for hubtest") cmd.Flags().StringVar(&appSecHost, "host", hubtest.DefaultAppsecHost, "Address to expose AppSec for hubtest")
cmd.Flags().BoolVar(&runAll, "all", false, "Run all tests") cmd.Flags().BoolVar(&all, "all", false, "Run all tests")
cmd.Flags().BoolVar(&reportSuccess, "report-success", false, "Report successful tests too (implied with json output)")
cmd.Flags().UintVar(&maxJobs, "max-jobs", maxJobs, "Run <num> batch")
return cmd return cmd
} }

22
cmd/crowdsec-cli/clihubtest/table.go
View file

@ -3,6 +3,7 @@ package clihubtest
import ( import (
"fmt" "fmt"
"io" "io"
"strconv"
"github.com/jedib0t/go-pretty/v6/text" "github.com/jedib0t/go-pretty/v6/text"
@ -11,22 +12,31 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/hubtest" "github.com/crowdsecurity/crowdsec/pkg/hubtest"
) )
func hubTestResultTable(out io.Writer, wantColor string, testResult map[string]bool) { func hubTestResultTable(out io.Writer, wantColor string, testMap map[string]*hubtest.HubTestItem, reportSuccess bool) {
t := cstable.NewLight(out, wantColor) t := cstable.NewLight(out, wantColor)
t.SetHeaders("Test", "Result") t.SetHeaders("Test", "Result", "Assertions")
t.SetHeaderAlignment(text.AlignLeft) t.SetHeaderAlignment(text.AlignLeft)
t.SetAlignment(text.AlignLeft) t.SetAlignment(text.AlignLeft)
for testName, success := range testResult { showTable := reportSuccess
for testName, test := range testMap {
status := emoji.CheckMarkButton status := emoji.CheckMarkButton
if !success { if !test.Success {
status = emoji.CrossMark status = emoji.CrossMark
showTable = true
} }
t.AddRow(testName, status) if !test.Success || reportSuccess {
t.AddRow(testName, status, strconv.Itoa(test.ParserAssert.NbAssert+test.ScenarioAssert.NbAssert))
}
} }
t.Render() if showTable {
t.Render()
} else {
fmt.Println("All tests passed, use --report-success for more details.")
}
} }
func hubTestListTable(out io.Writer, wantColor string, tests []*hubtest.HubTestItem) { func hubTestListTable(out io.Writer, wantColor string, tests []*hubtest.HubTestItem) {

6
pkg/dumps/parser_dump.go
View file

@ -35,6 +35,8 @@ type DumpOpts struct {
} }
func LoadParserDump(filepath string) (*ParserResults, error) { func LoadParserDump(filepath string) (*ParserResults, error) {
logger := log.WithField("file", filepath)
dumpData, err := os.Open(filepath) dumpData, err := os.Open(filepath)
if err != nil { if err != nil {
return nil, err return nil, err
@ -83,9 +85,9 @@ func LoadParserDump(filepath string) (*ParserResults, error) {
for idx, result := range pdump[lastStage][lastParser] { for idx, result := range pdump[lastStage][lastParser] {
if result.Evt.StrTime == "" { if result.Evt.StrTime == "" {
log.Warningf("Line %d/%d is missing evt.StrTime. It is most likely a mistake as it will prevent your logs to be processed in time-machine/forensic mode.", idx, len(pdump[lastStage][lastParser])) logger.Warningf("Line %d/%d is missing evt.StrTime. It is most likely a mistake as it will prevent your logs to be processed in time-machine/forensic mode.", idx, len(pdump[lastStage][lastParser]))
} else { } else {
log.Debugf("Line %d/%d has evt.StrTime set to '%s'", idx, len(pdump[lastStage][lastParser]), result.Evt.StrTime) logger.Debugf("Line %d/%d has evt.StrTime set to '%s'", idx, len(pdump[lastStage][lastParser]), result.Evt.StrTime)
} }
} }

17
pkg/hubtest/coverage.go
View file

@ -23,7 +23,7 @@ type Coverage struct {
PresentIn map[string]bool // poorman's set PresentIn map[string]bool // poorman's set
} }
func (h *HubTest) GetAppsecCoverage() ([]Coverage, error) { func (h *HubTest) GetAppsecCoverage(hubDir string) ([]Coverage, error) {
if len(h.HubIndex.GetItemMap(cwhub.APPSEC_RULES)) == 0 { if len(h.HubIndex.GetItemMap(cwhub.APPSEC_RULES)) == 0 {
return nil, errors.New("no appsec rules in hub index") return nil, errors.New("no appsec rules in hub index")
} }
@ -41,7 +41,7 @@ func (h *HubTest) GetAppsecCoverage() ([]Coverage, error) {
} }
// parser the expressions a-la-oneagain // parser the expressions a-la-oneagain
appsecTestConfigs, err := filepath.Glob(".appsec-tests/*/config.yaml") appsecTestConfigs, err := filepath.Glob(filepath.Join(hubDir, ".appsec-tests", "*", "config.yaml"))
if err != nil { if err != nil {
return nil, fmt.Errorf("while find appsec-tests config: %w", err) return nil, fmt.Errorf("while find appsec-tests config: %w", err)
} }
@ -57,7 +57,7 @@ func (h *HubTest) GetAppsecCoverage() ([]Coverage, error) {
err = yaml.Unmarshal(yamlFile, configFileData) err = yaml.Unmarshal(yamlFile, configFileData)
if err != nil { if err != nil {
return nil, fmt.Errorf("parsing: %v", err) return nil, fmt.Errorf("parsing: %w", err)
} }
for _, appsecRulesFile := range configFileData.AppsecRules { for _, appsecRulesFile := range configFileData.AppsecRules {
@ -70,7 +70,7 @@ func (h *HubTest) GetAppsecCoverage() ([]Coverage, error) {
err = yaml.Unmarshal(yamlFile, appsecRuleData) err = yaml.Unmarshal(yamlFile, appsecRuleData)
if err != nil { if err != nil {
return nil, fmt.Errorf("parsing: %v", err) return nil, fmt.Errorf("parsing: %w", err)
} }
appsecRuleName := appsecRuleData.Name appsecRuleName := appsecRuleData.Name
@ -87,7 +87,7 @@ func (h *HubTest) GetAppsecCoverage() ([]Coverage, error) {
return coverage, nil return coverage, nil
} }
func (h *HubTest) GetParsersCoverage() ([]Coverage, error) { func (h *HubTest) GetParsersCoverage(hubDir string) ([]Coverage, error) {
if len(h.HubIndex.GetItemMap(cwhub.PARSERS)) == 0 { if len(h.HubIndex.GetItemMap(cwhub.PARSERS)) == 0 {
return nil, errors.New("no parsers in hub index") return nil, errors.New("no parsers in hub index")
} }
@ -105,7 +105,7 @@ func (h *HubTest) GetParsersCoverage() ([]Coverage, error) {
} }
// parser the expressions a-la-oneagain // parser the expressions a-la-oneagain
passerts, err := filepath.Glob(".tests/*/parser.assert") passerts, err := filepath.Glob(filepath.Join(hubDir, ".tests", "*", "parser.assert"))
if err != nil { if err != nil {
return nil, fmt.Errorf("while find parser asserts: %w", err) return nil, fmt.Errorf("while find parser asserts: %w", err)
} }
@ -173,7 +173,7 @@ func (h *HubTest) GetParsersCoverage() ([]Coverage, error) {
return coverage, nil return coverage, nil
} }
func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) { func (h *HubTest) GetScenariosCoverage(hubDir string) ([]Coverage, error) {
if len(h.HubIndex.GetItemMap(cwhub.SCENARIOS)) == 0 { if len(h.HubIndex.GetItemMap(cwhub.SCENARIOS)) == 0 {
return nil, errors.New("no scenarios in hub index") return nil, errors.New("no scenarios in hub index")
} }
@ -191,7 +191,7 @@ func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) {
} }
// parser the expressions a-la-oneagain // parser the expressions a-la-oneagain
passerts, err := filepath.Glob(".tests/*/scenario.assert") passerts, err := filepath.Glob(filepath.Join(hubDir, ".tests", "*", "scenario.assert"))
if err != nil { if err != nil {
return nil, fmt.Errorf("while find scenario asserts: %w", err) return nil, fmt.Errorf("while find scenario asserts: %w", err)
} }
@ -259,6 +259,7 @@ func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) {
} }
} }
} }
file.Close() file.Close()
} }

10
pkg/hubtest/helpers.go Normal file
View file

@ -0,0 +1,10 @@
package hubtest
import (
"path/filepath"
)
func basename(params ...any) (any, error) {
s := params[0].(string)
return filepath.Base(s), nil
}

18
pkg/hubtest/hubtest.go
View file

@ -24,13 +24,13 @@ type HubTest struct {
TemplateAppsecProfilePath string TemplateAppsecProfilePath string
NucleiTargetHost string NucleiTargetHost string
AppSecHost string AppSecHost string
DataDir string // we share this one across tests, to avoid unnecessary downloads
HubIndex *cwhub.Hub HubIndex *cwhub.Hub
Tests []*HubTestItem Tests []*HubTestItem
} }
const ( const (
templateConfigFile = "template_config.yaml" templateConfigFile = "template_config2.yaml"
templateSimulationFile = "template_simulation.yaml" templateSimulationFile = "template_simulation.yaml"
templateProfileFile = "template_profiles.yaml" templateProfileFile = "template_profiles.yaml"
templateAcquisFile = "template_acquis.yaml" templateAcquisFile = "template_acquis.yaml"
@ -61,7 +61,7 @@ http:
func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecTest bool) (HubTest, error) { func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecTest bool) (HubTest, error) {
hubPath, err := filepath.Abs(hubPath) hubPath, err := filepath.Abs(hubPath)
if err != nil { if err != nil {
return HubTest{}, fmt.Errorf("can't get absolute path of hub: %+v", err) return HubTest{}, fmt.Errorf("can't get absolute path of hub: %w", err)
} }
// we can't use hubtest without the hub // we can't use hubtest without the hub
@ -139,9 +139,15 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecT
return HubTest{}, err return HubTest{}, err
} }
dataDir := filepath.Join(hubPath, ".cache", "data")
if err = os.MkdirAll(dataDir, 0o700); err != nil {
return HubTest{}, fmt.Errorf("while creating data dir: %w", err)
}
return HubTest{ return HubTest{
CrowdSecPath: crowdsecPath, CrowdSecPath: crowdsecPath,
CscliPath: cscliPath, CscliPath: cscliPath,
DataDir: dataDir,
HubPath: hubPath, HubPath: hubPath,
HubTestPath: HubTestPath, HubTestPath: HubTestPath,
HubIndexFile: hubIndexFile, HubIndexFile: hubIndexFile,
@ -155,7 +161,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecT
func (h *HubTest) LoadTestItem(name string) (*HubTestItem, error) { func (h *HubTest) LoadTestItem(name string) (*HubTestItem, error) {
HubTestItem := &HubTestItem{} HubTestItem := &HubTestItem{}
testItem, err := NewTest(name, h) testItem, err := NewTest(name, h, h.DataDir)
if err != nil { if err != nil {
return HubTestItem, err return HubTestItem, err
} }

135
pkg/hubtest/hubtest_item.go
View file

@ -4,11 +4,13 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"io/fs"
"net/url" "net/url"
"os" "os"
"os/exec" "os/exec"
"path/filepath" "path/filepath"
"strings" "strings"
"sync"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
@ -19,6 +21,8 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/parser" "github.com/crowdsecurity/crowdsec/pkg/parser"
) )
var downloadMutex sync.Mutex
type HubTestItemConfig struct { type HubTestItemConfig struct {
Parsers []string `yaml:"parsers,omitempty"` Parsers []string `yaml:"parsers,omitempty"`
Scenarios []string `yaml:"scenarios,omitempty"` Scenarios []string `yaml:"scenarios,omitempty"`
@ -31,6 +35,7 @@ type HubTestItemConfig struct {
Labels map[string]string `yaml:"labels,omitempty"` Labels map[string]string `yaml:"labels,omitempty"`
IgnoreParsers bool `yaml:"ignore_parsers,omitempty"` // if we test a scenario, we don't want to assert on Parser IgnoreParsers bool `yaml:"ignore_parsers,omitempty"` // if we test a scenario, we don't want to assert on Parser
OverrideStatics []parser.ExtraField `yaml:"override_statics,omitempty"` // Allow to override statics. Executed before s00 OverrideStatics []parser.ExtraField `yaml:"override_statics,omitempty"` // Allow to override statics. Executed before s00
OwnDataDir bool `yaml:"own_data_dir,omitempty"` // Don't share dataDir with the other tests
} }
type HubTestItem struct { type HubTestItem struct {
@ -41,6 +46,7 @@ type HubTestItem struct {
CscliPath string CscliPath string
RuntimePath string RuntimePath string
RuntimeDBDir string
RuntimeHubPath string RuntimeHubPath string
RuntimeDataPath string RuntimeDataPath string
RuntimePatternsPath string RuntimePatternsPath string
@ -95,7 +101,7 @@ const (
DefaultAppsecHost = "127.0.0.1:4241" DefaultAppsecHost = "127.0.0.1:4241"
) )
func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { func NewTest(name string, hubTest *HubTest, dataDir string) (*HubTestItem, error) {
testPath := filepath.Join(hubTest.HubTestPath, name) testPath := filepath.Join(hubTest.HubTestPath, name)
runtimeFolder := filepath.Join(testPath, "runtime") runtimeFolder := filepath.Join(testPath, "runtime")
runtimeHubFolder := filepath.Join(runtimeFolder, "hub") runtimeHubFolder := filepath.Join(runtimeFolder, "hub")
@ -121,6 +127,15 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
scenarioAssertFilePath := filepath.Join(testPath, ScenarioAssertFileName) scenarioAssertFilePath := filepath.Join(testPath, ScenarioAssertFileName)
ScenarioAssert := NewScenarioAssert(scenarioAssertFilePath) ScenarioAssert := NewScenarioAssert(scenarioAssertFilePath)
// force own_data_dir for backard compatibility
if name == "magento-ccs-by-as" || name == "magento-ccs-by-country" || name == "geoip-enrich" {
configFileData.OwnDataDir = true
}
if configFileData.OwnDataDir {
dataDir = filepath.Join(runtimeFolder, "data")
}
return &HubTestItem{ return &HubTestItem{
Name: name, Name: name,
Path: testPath, Path: testPath,
@ -128,8 +143,8 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
CscliPath: hubTest.CscliPath, CscliPath: hubTest.CscliPath,
RuntimePath: filepath.Join(testPath, "runtime"), RuntimePath: filepath.Join(testPath, "runtime"),
RuntimeHubPath: runtimeHubFolder, RuntimeHubPath: runtimeHubFolder,
RuntimeDataPath: filepath.Join(runtimeFolder, "data"),
RuntimePatternsPath: filepath.Join(runtimeFolder, "patterns"), RuntimePatternsPath: filepath.Join(runtimeFolder, "patterns"),
RuntimeDBDir: filepath.Join(runtimeFolder, "data"),
RuntimeConfigFilePath: filepath.Join(runtimeFolder, "config.yaml"), RuntimeConfigFilePath: filepath.Join(runtimeFolder, "config.yaml"),
RuntimeProfileFilePath: filepath.Join(runtimeFolder, "profiles.yaml"), RuntimeProfileFilePath: filepath.Join(runtimeFolder, "profiles.yaml"),
RuntimeSimulationFilePath: filepath.Join(runtimeFolder, "simulation.yaml"), RuntimeSimulationFilePath: filepath.Join(runtimeFolder, "simulation.yaml"),
@ -142,7 +157,7 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
HubDir: runtimeHubFolder, HubDir: runtimeHubFolder,
HubIndexFile: hubTest.HubIndexFile, HubIndexFile: hubTest.HubIndexFile,
InstallDir: runtimeFolder, InstallDir: runtimeFolder,
InstallDataDir: filepath.Join(runtimeFolder, "data"), InstallDataDir: dataDir,
}, },
Config: configFileData, Config: configFileData,
HubPath: hubTest.HubPath, HubPath: hubTest.HubPath,
@ -176,7 +191,7 @@ func (t *HubTestItem) installHubItems(names []string, installFunc func(string) e
return nil return nil
} }
func (t *HubTestItem) InstallHub() error { func (t *HubTestItem) InstallHub(ctx context.Context) error {
if err := t.installHubItems(t.Config.Parsers, t.installParser); err != nil { if err := t.installHubItems(t.Config.Parsers, t.installParser); err != nil {
return err return err
} }
@ -221,12 +236,14 @@ func (t *HubTestItem) InstallHub() error {
return err return err
} }
ctx := context.Background() // prevent concurrent downloads of the same file
downloadMutex.Lock()
defer downloadMutex.Unlock()
// install data for parsers if needed // install data for parsers if needed
for _, item := range hub.GetInstalledByType(cwhub.PARSERS, true) { for _, item := range hub.GetInstalledByType(cwhub.PARSERS, true) {
if _, err := hubops.DownloadDataIfNeeded(ctx, hub, item, true); err != nil { if _, err := hubops.DownloadDataIfNeeded(ctx, hub, item, false); err != nil {
return fmt.Errorf("unable to download data for parser '%s': %+v", item.Name, err) return fmt.Errorf("unable to download data for parser '%s': %w", item.Name, err)
} }
log.Debugf("parser '%s' installed successfully in runtime environment", item.Name) log.Debugf("parser '%s' installed successfully in runtime environment", item.Name)
@ -234,8 +251,8 @@ func (t *HubTestItem) InstallHub() error {
// install data for scenarios if needed // install data for scenarios if needed
for _, item := range hub.GetInstalledByType(cwhub.SCENARIOS, true) { for _, item := range hub.GetInstalledByType(cwhub.SCENARIOS, true) {
if _, err := hubops.DownloadDataIfNeeded(ctx, hub, item, true); err != nil { if _, err := hubops.DownloadDataIfNeeded(ctx, hub, item, false); err != nil {
return fmt.Errorf("unable to download data for parser '%s': %+v", item.Name, err) return fmt.Errorf("unable to download data for parser '%s': %w", item.Name, err)
} }
log.Debugf("scenario '%s' installed successfully in runtime environment", item.Name) log.Debugf("scenario '%s' installed successfully in runtime environment", item.Name)
@ -243,8 +260,8 @@ func (t *HubTestItem) InstallHub() error {
// install data for postoverflows if needed // install data for postoverflows if needed
for _, item := range hub.GetInstalledByType(cwhub.POSTOVERFLOWS, true) { for _, item := range hub.GetInstalledByType(cwhub.POSTOVERFLOWS, true) {
if _, err := hubops.DownloadDataIfNeeded(ctx, hub, item, true); err != nil { if _, err := hubops.DownloadDataIfNeeded(ctx, hub, item, false); err != nil {
return fmt.Errorf("unable to download data for parser '%s': %+v", item.Name, err) return fmt.Errorf("unable to download data for parser '%s': %w", item.Name, err)
} }
log.Debugf("postoverflow '%s' installed successfully in runtime environment", item.Name) log.Debugf("postoverflow '%s' installed successfully in runtime environment", item.Name)
@ -253,49 +270,61 @@ func (t *HubTestItem) InstallHub() error {
return nil return nil
} }
func (t *HubTestItem) Clean() error { func (t *HubTestItem) Clean() {
return os.RemoveAll(t.RuntimePath) if err := os.RemoveAll(t.ResultsPath); err != nil {
if !errors.Is(err, fs.ErrNotExist) {
log.Errorf("while cleaning %s: %s", t.Name, err.Error())
}
}
if err := os.RemoveAll(t.RuntimePath); err != nil {
if !errors.Is(err, fs.ErrNotExist) {
log.Errorf("while cleaning %s: %s", t.Name, err.Error())
}
}
} }
func (t *HubTestItem) RunWithNucleiTemplate() error { func (t *HubTestItem) RunWithNucleiTemplate() error {
crowdsecLogFile := fmt.Sprintf("%s/log/crowdsec.log", t.RuntimePath)
testPath := filepath.Join(t.HubTestPath, t.Name) testPath := filepath.Join(t.HubTestPath, t.Name)
if _, err := os.Stat(testPath); os.IsNotExist(err) { if _, err := os.Stat(testPath); os.IsNotExist(err) {
return fmt.Errorf("test '%s' doesn't exist in '%s', exiting", t.Name, t.HubTestPath) return fmt.Errorf("test '%s' doesn't exist in '%s', exiting", t.Name, t.HubTestPath)
} }
if err := os.Chdir(testPath); err != nil { crowdsecLogFile := fmt.Sprintf("%s/log/crowdsec.log", t.RuntimePath)
return fmt.Errorf("can't 'cd' to '%s': %w", testPath, err)
}
// machine add // machine add
cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--force", "--auto"} cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--force", "--auto"}
cscliRegisterCmd := exec.Command(t.CscliPath, cmdArgs...) cscliRegisterCmd := exec.Command(t.CscliPath, cmdArgs...)
cscliRegisterCmd.Dir = testPath
cscliRegisterCmd.Env = []string{"TESTDIR="+testPath, "DATADIR="+t.RuntimeHubConfig.InstallDataDir, "TZ=UTC"}
output, err := cscliRegisterCmd.CombinedOutput() output, err := cscliRegisterCmd.CombinedOutput()
if err != nil { if err != nil {
if !strings.Contains(string(output), "unable to create machine: user 'testMachine': user already exist") { if !strings.Contains(string(output), "unable to create machine: user 'testMachine': user already exist") {
fmt.Println(string(output)) fmt.Println(string(output))
return fmt.Errorf("fail to run '%s' for test '%s': %v", cscliRegisterCmd.String(), t.Name, err) return fmt.Errorf("fail to run '%s' for test '%s': %w", cscliRegisterCmd.String(), t.Name, err)
} }
} }
// hardcode bouncer key // hardcode bouncer key
cmdArgs = []string{"-c", t.RuntimeConfigFilePath, "bouncers", "add", "appsectests", "-k", TestBouncerApiKey} cmdArgs = []string{"-c", t.RuntimeConfigFilePath, "bouncers", "add", "appsectests", "-k", TestBouncerApiKey}
cscliBouncerCmd := exec.Command(t.CscliPath, cmdArgs...) cscliBouncerCmd := exec.Command(t.CscliPath, cmdArgs...)
cscliBouncerCmd.Dir = testPath
cscliBouncerCmd.Env = []string{"TESTDIR="+testPath, "DATADIR="+t.RuntimeHubConfig.InstallDataDir, "TZ=UTC"}
output, err = cscliBouncerCmd.CombinedOutput() output, err = cscliBouncerCmd.CombinedOutput()
if err != nil { if err != nil {
if !strings.Contains(string(output), "unable to create bouncer: bouncer appsectests already exists") { if !strings.Contains(string(output), "unable to create bouncer: bouncer appsectests already exists") {
fmt.Println(string(output)) fmt.Println(string(output))
return fmt.Errorf("fail to run '%s' for test '%s': %v", cscliRegisterCmd.String(), t.Name, err) return fmt.Errorf("fail to run '%s' for test '%s': %w", cscliRegisterCmd.String(), t.Name, err)
} }
} }
// start crowdsec service // start crowdsec service
cmdArgs = []string{"-c", t.RuntimeConfigFilePath} cmdArgs = []string{"-c", t.RuntimeConfigFilePath}
crowdsecDaemon := exec.Command(t.CrowdSecPath, cmdArgs...) crowdsecDaemon := exec.Command(t.CrowdSecPath, cmdArgs...)
crowdsecDaemon.Dir = testPath
crowdsecDaemon.Env = []string{"TESTDIR="+testPath, "DATADIR="+t.RuntimeHubConfig.InstallDataDir, "TZ=UTC"}
crowdsecDaemon.Start() crowdsecDaemon.Start()
@ -382,59 +411,16 @@ func createDirs(dirs []string) error {
return nil return nil
} }
func (t *HubTestItem) RunWithLogFile(patternDir string) error { func (t *HubTestItem) RunWithLogFile() error {
testPath := filepath.Join(t.HubTestPath, t.Name) testPath := filepath.Join(t.HubTestPath, t.Name)
if _, err := os.Stat(testPath); os.IsNotExist(err) { if _, err := os.Stat(testPath); os.IsNotExist(err) {
return fmt.Errorf("test '%s' doesn't exist in '%s', exiting", t.Name, t.HubTestPath) return fmt.Errorf("test '%s' doesn't exist in '%s', exiting", t.Name, t.HubTestPath)
} }
currentDir, err := os.Getwd() // xx logFile := filepath.Join(testPath, t.Config.LogFile)
if err != nil {
return fmt.Errorf("can't get current directory: %+v", err)
}
// create runtime, data, hub folders
if err = createDirs([]string{t.RuntimePath, t.RuntimeDataPath, t.RuntimeHubPath, t.ResultsPath}); err != nil {
return err
}
if err = Copy(t.HubIndexFile, filepath.Join(t.RuntimeHubPath, ".index.json")); err != nil {
return fmt.Errorf("unable to copy .index.json file in '%s': %w", filepath.Join(t.RuntimeHubPath, ".index.json"), err)
}
// copy template config file to runtime folder
if err = Copy(t.TemplateConfigPath, t.RuntimeConfigFilePath); err != nil {
return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateConfigPath, t.RuntimeConfigFilePath, err)
}
// copy template profile file to runtime folder
if err = Copy(t.TemplateProfilePath, t.RuntimeProfileFilePath); err != nil {
return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateProfilePath, t.RuntimeProfileFilePath, err)
}
// copy template simulation file to runtime folder
if err = Copy(t.TemplateSimulationPath, t.RuntimeSimulationFilePath); err != nil {
return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateSimulationPath, t.RuntimeSimulationFilePath, err)
}
// copy template patterns folder to runtime folder
if err = CopyDir(patternDir, t.RuntimePatternsPath); err != nil {
return fmt.Errorf("unable to copy 'patterns' from '%s' to '%s': %w", patternDir, t.RuntimePatternsPath, err)
}
// install the hub in the runtime folder
if err = t.InstallHub(); err != nil {
return fmt.Errorf("unable to install hub in '%s': %w", t.RuntimeHubPath, err)
}
logFile := t.Config.LogFile
logType := t.Config.LogType logType := t.Config.LogType
dsn := fmt.Sprintf("file://%s", logFile) dsn := fmt.Sprintf("file://%s", logFile)
if err = os.Chdir(testPath); err != nil {
return fmt.Errorf("can't 'cd' to '%s': %w", testPath, err)
}
logFileStat, err := os.Stat(logFile) logFileStat, err := os.Stat(logFile)
if err != nil { if err != nil {
return fmt.Errorf("unable to stat log file '%s': %w", logFile, err) return fmt.Errorf("unable to stat log file '%s': %w", logFile, err)
@ -446,6 +432,9 @@ func (t *HubTestItem) RunWithLogFile(patternDir string) error {
cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--force", "--auto"} cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--force", "--auto"}
cscliRegisterCmd := exec.Command(t.CscliPath, cmdArgs...) cscliRegisterCmd := exec.Command(t.CscliPath, cmdArgs...)
cscliRegisterCmd.Dir = testPath
cscliRegisterCmd.Env = []string{"TESTDIR="+testPath, "DATADIR="+t.RuntimeHubConfig.InstallDataDir, "TZ=UTC"}
log.Debugf("%s", cscliRegisterCmd.String()) log.Debugf("%s", cscliRegisterCmd.String())
output, err := cscliRegisterCmd.CombinedOutput() output, err := cscliRegisterCmd.CombinedOutput()
@ -464,6 +453,9 @@ func (t *HubTestItem) RunWithLogFile(patternDir string) error {
} }
crowdsecCmd := exec.Command(t.CrowdSecPath, cmdArgs...) crowdsecCmd := exec.Command(t.CrowdSecPath, cmdArgs...)
crowdsecCmd.Dir = testPath
crowdsecCmd.Env = []string{"TESTDIR="+testPath, "DATADIR="+t.RuntimeHubConfig.InstallDataDir, "TZ=UTC"}
log.Debugf("%s", crowdsecCmd.String()) log.Debugf("%s", crowdsecCmd.String())
output, err = crowdsecCmd.CombinedOutput() output, err = crowdsecCmd.CombinedOutput()
@ -475,10 +467,6 @@ func (t *HubTestItem) RunWithLogFile(patternDir string) error {
return fmt.Errorf("fail to run '%s' for test '%s': %v", crowdsecCmd.String(), t.Name, err) return fmt.Errorf("fail to run '%s' for test '%s': %v", crowdsecCmd.String(), t.Name, err)
} }
if err := os.Chdir(currentDir); err != nil {
return fmt.Errorf("can't 'cd' to '%s': %w", currentDir, err)
}
// assert parsers // assert parsers
if !t.Config.IgnoreParsers { if !t.Config.IgnoreParsers {
_, err := os.Stat(t.ParserAssert.File) _, err := os.Stat(t.ParserAssert.File)
@ -506,6 +494,7 @@ func (t *HubTestItem) RunWithLogFile(patternDir string) error {
t.ParserAssert.AutoGenAssert = true t.ParserAssert.AutoGenAssert = true
} else { } else {
if err := t.ParserAssert.AssertFile(t.ParserResultFile); err != nil { if err := t.ParserAssert.AssertFile(t.ParserResultFile); err != nil {
// TODO: no error - should not prevent running the other tests
return fmt.Errorf("unable to run assertion on file '%s': %w", t.ParserResultFile, err) return fmt.Errorf("unable to run assertion on file '%s': %w", t.ParserResultFile, err)
} }
} }
@ -564,14 +553,14 @@ func (t *HubTestItem) RunWithLogFile(patternDir string) error {
return nil return nil
} }
func (t *HubTestItem) Run(patternDir string) error { func (t *HubTestItem) Run(ctx context.Context, patternDir string) error {
var err error var err error
t.Success = false t.Success = false
t.ErrorsList = make([]string, 0) t.ErrorsList = make([]string, 0)
// create runtime, data, hub, result folders // create runtime, data, hub, result folders
if err = createDirs([]string{t.RuntimePath, t.RuntimeDataPath, t.RuntimeHubPath, t.ResultsPath}); err != nil { if err = createDirs([]string{t.RuntimePath, t.RuntimeDBDir, t.RuntimeHubConfig.InstallDataDir, t.RuntimeHubPath, t.ResultsPath}); err != nil {
return err return err
} }
@ -625,12 +614,12 @@ func (t *HubTestItem) Run(patternDir string) error {
} }
// install the hub in the runtime folder // install the hub in the runtime folder
if err = t.InstallHub(); err != nil { if err = t.InstallHub(ctx); err != nil {
return fmt.Errorf("unable to install hub in '%s': %w", t.RuntimeHubPath, err) return fmt.Errorf("unable to install hub in '%s': %w", t.RuntimeHubPath, err)
} }
if t.Config.LogFile != "" { if t.Config.LogFile != "" {
return t.RunWithLogFile(patternDir) return t.RunWithLogFile()
} }
if t.Config.NucleiTemplate != "" { if t.Config.NucleiTemplate != "" {

41
pkg/hubtest/parser_assert.go
View file

@ -61,7 +61,7 @@ func (p *ParserAssert) AutoGenFromFile(filename string) (string, error) {
func (p *ParserAssert) LoadTest(filename string) error { func (p *ParserAssert) LoadTest(filename string) error {
parserDump, err := dumps.LoadParserDump(filename) parserDump, err := dumps.LoadParserDump(filename)
if err != nil { if err != nil {
return fmt.Errorf("loading parser dump file: %+v", err) return fmt.Errorf("loading parser dump file: %w", err)
} }
p.TestData = parserDump p.TestData = parserDump
@ -93,7 +93,7 @@ func (p *ParserAssert) AssertFile(testFile string) error {
ok, err := p.Run(scanner.Text()) ok, err := p.Run(scanner.Text())
if err != nil { if err != nil {
return fmt.Errorf("unable to run assert '%s': %+v", scanner.Text(), err) return fmt.Errorf("unable to run assert '%s': %w", scanner.Text(), err)
} }
p.NbAssert++ p.NbAssert++
@ -151,26 +151,43 @@ func (p *ParserAssert) AssertFile(testFile string) error {
return nil return nil
} }
func basenameShim(expression string) string {
if strings.Contains(expression, "datasource_path") && !strings.Contains(expression, "basename(") {
// match everything before == and wrap it with basename()
match := strings.Split(expression, "==")
return fmt.Sprintf("basename(%s) == %s", match[0], match[1])
}
return expression
}
func (p *ParserAssert) RunExpression(expression string) (interface{}, error) { func (p *ParserAssert) RunExpression(expression string) (interface{}, error) {
// debug doesn't make much sense with the ability to evaluate "on the fly" // debug doesn't make much sense with the ability to evaluate "on the fly"
// var debugFilter *exprhelpers.ExprDebugger // var debugFilter *exprhelpers.ExprDebugger
var output interface{} var output any
env := map[string]interface{}{"results": *p.TestData} logger := log.WithField("file", p.File)
runtimeFilter, err := expr.Compile(expression, exprhelpers.GetExprOptions(env)...) env := map[string]any{"results": *p.TestData}
opts := exprhelpers.GetExprOptions(env)
opts = append(opts, expr.Function("basename", basename, new(func (string) string)))
// wrap with basename() in case of datasource_path, for backward compatibility
expression = basenameShim(expression)
runtimeFilter, err := expr.Compile(expression, opts...)
if err != nil { if err != nil {
log.Errorf("failed to compile '%s' : %s", expression, err) logger.Errorf("failed to compile '%s': %s", expression, err)
return output, err return output, err
} }
// dump opcode in trace level // dump opcode in trace level
log.Tracef("%s", runtimeFilter.Disassemble()) logger.Tracef("%s", runtimeFilter.Disassemble())
output, err = expr.Run(runtimeFilter, env) output, err = expr.Run(runtimeFilter, env)
if err != nil { if err != nil {
log.Warningf("running : %s", expression) logger.Warningf("running : %s", expression)
log.Warningf("runtime error : %s", err) logger.Warningf("runtime error: %s", err)
return output, fmt.Errorf("while running expression %s: %w", expression, err) return output, fmt.Errorf("while running expression %s: %w", expression, err)
} }
@ -252,7 +269,11 @@ func (p *ParserAssert) AutoGenParserAssert() string {
continue continue
} }
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Meta["%s"] == "%s"`+"\n", stage, parser, pidx, mkey, Escape(mval)) if mkey == "datasource_path" {
ret += fmt.Sprintf(`basename(results["%s"]["%s"][%d].Evt.Meta["%s"]) == "%s"`+"\n", stage, parser, pidx, mkey, Escape(mval))
} else {
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Meta["%s"] == "%s"`+"\n", stage, parser, pidx, mkey, Escape(mval))
}
} }
for _, ekey := range maptools.SortedKeys(result.Evt.Enriched) { for _, ekey := range maptools.SortedKeys(result.Evt.Enriched) {

37
pkg/hubtest/scenario_assert.go
View file

@ -6,6 +6,7 @@ import (
"fmt" "fmt"
"io" "io"
"os" "os"
"path/filepath"
"sort" "sort"
"strings" "strings"
@ -59,7 +60,7 @@ func (s *ScenarioAssert) AutoGenFromFile(filename string) (string, error) {
func (s *ScenarioAssert) LoadTest(filename string, bucketpour string) error { func (s *ScenarioAssert) LoadTest(filename string, bucketpour string) error {
bucketDump, err := LoadScenarioDump(filename) bucketDump, err := LoadScenarioDump(filename)
if err != nil { if err != nil {
return fmt.Errorf("loading scenario dump file '%s': %+v", filename, err) return fmt.Errorf("loading scenario dump file '%s': %w", filename, err)
} }
s.TestData = bucketDump s.TestData = bucketDump
@ -67,7 +68,7 @@ func (s *ScenarioAssert) LoadTest(filename string, bucketpour string) error {
if bucketpour != "" { if bucketpour != "" {
pourDump, err := dumps.LoadBucketPourDump(bucketpour) pourDump, err := dumps.LoadBucketPourDump(bucketpour)
if err != nil { if err != nil {
return fmt.Errorf("loading bucket pour dump file '%s': %+v", filename, err) return fmt.Errorf("loading bucket pour dump file '%s': %w", filename, err)
} }
s.PourData = pourDump s.PourData = pourDump
@ -100,7 +101,7 @@ func (s *ScenarioAssert) AssertFile(testFile string) error {
ok, err := s.Run(scanner.Text()) ok, err := s.Run(scanner.Text())
if err != nil { if err != nil {
return fmt.Errorf("unable to run assert '%s': %+v", scanner.Text(), err) return fmt.Errorf("unable to run assert '%s': %w", scanner.Text(), err)
} }
s.NbAssert++ s.NbAssert++
@ -156,28 +157,34 @@ func (s *ScenarioAssert) AssertFile(testFile string) error {
return nil return nil
} }
func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) { func (s *ScenarioAssert) RunExpression(expression string) (any, error) {
// debug doesn't make much sense with the ability to evaluate "on the fly" // debug doesn't make much sense with the ability to evaluate "on the fly"
// var debugFilter *exprhelpers.ExprDebugger // var debugFilter *exprhelpers.ExprDebugger
var output interface{} var output any
env := map[string]interface{}{"results": *s.TestData} logger := log.WithField("file", s.File)
runtimeFilter, err := expr.Compile(expression, exprhelpers.GetExprOptions(env)...) env := map[string]any{"results": *s.TestData}
opts := exprhelpers.GetExprOptions(env)
opts = append(opts, expr.Function("basename", basename, new(func (string) string)))
expression = basenameShim(expression)
runtimeFilter, err := expr.Compile(expression, opts...)
if err != nil { if err != nil {
return nil, err return nil, err
} }
// if debugFilter, err = exprhelpers.NewDebugger(assert, expr.Env(env)); err != nil { // if debugFilter, err = exprhelpers.NewDebugger(assert, expr.Env(env)); err != nil {
// log.Warningf("Failed building debugher for %s : %s", assert, err) // logger.Warningf("Failed building debugher for %s : %s", assert, err)
// } // }
// dump opcode in trace level // dump opcode in trace level
log.Tracef("%s", runtimeFilter.Disassemble()) logger.Tracef("%s", runtimeFilter.Disassemble())
output, err = expr.Run(runtimeFilter, map[string]interface{}{"results": *s.TestData}) output, err = expr.Run(runtimeFilter, map[string]any{"results": *s.TestData})
if err != nil { if err != nil {
log.Warningf("running : %s", expression) logger.Warningf("running : %s", expression)
log.Warningf("runtime error : %s", err) logger.Warningf("runtime error : %s", err)
return nil, fmt.Errorf("while running expression %s: %w", expression, err) return nil, fmt.Errorf("while running expression %s: %w", expression, err)
} }
@ -228,7 +235,11 @@ func (s *ScenarioAssert) AutoGenScenarioAssert() string {
for evtIndex, evt := range event.Overflow.Alert.Events { for evtIndex, evt := range event.Overflow.Alert.Events {
for _, meta := range evt.Meta { for _, meta := range evt.Meta {
ret += fmt.Sprintf(`results[%d].Overflow.Alert.Events[%d].GetMeta("%s") == "%s"`+"\n", eventIndex, evtIndex, meta.Key, Escape(meta.Value)) if meta.Key == "datasource_path" {
ret += fmt.Sprintf(`basename(results[%d].Overflow.Alert.Events[%d].GetMeta("%s")) == "%s"`+"\n", eventIndex, evtIndex, meta.Key, Escape(filepath.Base(meta.Value)))
} else {
ret += fmt.Sprintf(`results[%d].Overflow.Alert.Events[%d].GetMeta("%s") == "%s"`+"\n", eventIndex, evtIndex, meta.Key, Escape(meta.Value))
}
} }
} }

1
test/bats.mk
View file

@ -103,7 +103,6 @@ bats-test: bats-environment ## Run functional tests
$(TEST_DIR)/run-tests $(TEST_DIR)/bats $(TEST_DIR)/run-tests $(TEST_DIR)/bats
bats-test-hub: bats-environment bats-check-requirements ## Run all hub tests bats-test-hub: bats-environment bats-check-requirements ## Run all hub tests
@$(TEST_DIR)/bin/generate-hub-tests
$(TEST_DIR)/run-tests $(TEST_DIR)/dyn-bats $(TEST_DIR)/run-tests $(TEST_DIR)/dyn-bats
# Not failproof but they can catch bugs and improve learning of sh/bash # Not failproof but they can catch bugs and improve learning of sh/bash

9
test/bin/collect-hub-coverage
View file

@ -12,13 +12,16 @@ THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
# shellcheck disable=SC1091 # shellcheck disable=SC1091
. "${THIS_DIR}/../.environment.sh" . "${THIS_DIR}/../.environment.sh"
hubdir="${LOCAL_DIR}/hub-tests"
coverage() { coverage() {
"${CSCLI}" --crowdsec "${CROWDSEC}" --cscli "${CSCLI}" hubtest coverage --"$1" --percent "${CSCLI}" --crowdsec "${CROWDSEC}" --cscli "${CSCLI}" hubtest coverage --"$1" --percent
} }
cd "${hubdir}" || die "Could not find hub test results" hubdir="${LOCAL_DIR}/hub-tests"
hubdir="${1:-${hubdir}}"
[[ -d "${hubdir}" ]] || die "Could not find hub test results in $hubdir"
cd "${hubdir}" || die "Could not find hub test results in $hubdir"
shopt -s inherit_errexit shopt -s inherit_errexit

19
test/bin/generate-hub-tests
View file

@ -1,19 +0,0 @@
#!/usr/bin/env bash
set -eu
# shellcheck disable=SC1007
THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
# shellcheck disable=SC1091
. "${THIS_DIR}/../.environment.sh"
"${TEST_DIR}/instance-data" load
hubdir="${LOCAL_DIR}/hub-tests"
git clone --depth 1 https://github.com/crowdsecurity/hub.git "${hubdir}" >/dev/null 2>&1 || (cd "${hubdir}"; git pull)
echo "Generating hub tests..."
python3 "$THIS_DIR/generate-hub-tests.py" \
<("${CSCLI}" --crowdsec "${CROWDSEC}" --cscli "${CSCLI}" hubtest --hub "${hubdir}" list -o json) \
"${TEST_DIR}/dyn-bats/"

63
test/bin/generate-hub-tests.py
View file

@ -1,63 +0,0 @@
#!/usr/bin/env python3
import json
import pathlib
import os
import sys
import textwrap
test_header = """
set -u
setup_file() {
load "../lib/setup_file.sh"
}
teardown_file() {
load "../lib/teardown_file.sh"
}
setup() {
load "../lib/setup.sh"
}
"""
def write_chunk(target_dir, n, chunk):
with open(target_dir / f"hub-{n}.bats", "w") as f:
f.write(test_header)
for test in chunk:
cscli = os.environ['CSCLI']
crowdsec = os.environ['CROWDSEC']
testname = test['Name']
hubdir = os.environ['LOCAL_DIR'] + '/hub-tests'
f.write(textwrap.dedent(f"""
@test "{testname}" {{
run "{cscli}" \\
--crowdsec "{crowdsec}" \\
--cscli "{cscli}" \\
--hub "{hubdir}" \\
hubtest run "{testname}" \\
--clean
echo "$output"
assert_success
}}
"""))
def main():
hubtests_json = sys.argv[1]
target_dir = sys.argv[2]
with open(hubtests_json) as f:
j = json.load(f)
chunk_size = len(j) // 3 + 1
n = 1
for i in range(0, len(j), chunk_size):
chunk = j[i:i + chunk_size]
write_chunk(pathlib.Path(target_dir), n, chunk)
n += 1
if __name__ == "__main__":
main()