diff --git a/.github/workflows/bats-hub.yml b/.github/workflows/bats-hub.yml index 42f1252c8..e2f47c414 100644 --- a/.github/workflows/bats-hub.yml +++ b/.github/workflows/bats-hub.yml @@ -10,9 +10,6 @@ on: jobs: build: - strategy: - matrix: - test-file: ["hub-1.bats", "hub-2.bats", "hub-3.bats"] name: "Functional tests" runs-on: ubuntu-latest @@ -46,11 +43,14 @@ jobs: - name: "Run hub tests" run: | - ./test/bin/generate-hub-tests - ./test/run-tests ./test/dyn-bats/${{ matrix.test-file }} --formatter $(pwd)/test/lib/color-formatter + PATH=$(pwd)/test/local/bin:$PATH + ./test/instance-data load + git clone --depth 1 https://github.com/crowdsecurity/hub.git ./hub + cd ./hub + cscli hubtest run --all --clean --max-jobs 8 - name: "Collect hub coverage" - run: ./test/bin/collect-hub-coverage >> $GITHUB_ENV + run: ./test/bin/collect-hub-coverage ./hub >> $GITHUB_ENV - name: "Create Parsers badge" uses: schneegans/dynamic-badges-action@v1.7.0 diff --git a/cmd/crowdsec-cli/clihubtest/clean.go b/cmd/crowdsec-cli/clihubtest/clean.go index 912b8838b..e18ae2fae 100644 --- a/cmd/crowdsec-cli/clihubtest/clean.go +++ b/cmd/crowdsec-cli/clihubtest/clean.go @@ -1,33 +1,55 @@ package clihubtest import ( + "errors" "fmt" "github.com/spf13/cobra" - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/args" + "github.com/crowdsecurity/crowdsec/pkg/hubtest" ) func (cli *cliHubTest) newCleanCmd() *cobra.Command { + var all bool + cmd := &cobra.Command{ Use: "clean", Short: "clean [test_name]", - Args: args.MinimumNArgs(1), DisableAutoGenTag: true, RunE: func(_ *cobra.Command, args []string) error { - for _, testName := range args { - test, err := hubPtr.LoadTestItem(testName) - if err != nil { - return fmt.Errorf("unable to load test '%s': %w", testName, err) + if !all && len(args) == 0 { + return errors.New("please provide test to run or --all flag") + } + + fmt.Println("Cleaning test data...") + + tests := []*hubtest.HubTestItem{} + + if all { + if err := hubPtr.LoadAllTests(); err != nil { + return fmt.Errorf("unable to load all tests: %w", err) } - if err := test.Clean(); err != nil { - return fmt.Errorf("unable to clean test '%s' env: %w", test.Name, err) + + tests = hubPtr.Tests + } else { + for _, testName := range args { + test, err := hubPtr.LoadTestItem(testName) + if err != nil { + return fmt.Errorf("unable to load test '%s': %w", testName, err) + } + tests = append(tests, test) } } + for _, test := range tests { + test.Clean() + } + return nil }, } + cmd.Flags().BoolVar(&all, "all", false, "Run all tests") + return cmd } diff --git a/cmd/crowdsec-cli/clihubtest/coverage.go b/cmd/crowdsec-cli/clihubtest/coverage.go index ee840f81e..d4f244071 100644 --- a/cmd/crowdsec-cli/clihubtest/coverage.go +++ b/cmd/crowdsec-cli/clihubtest/coverage.go @@ -14,12 +14,12 @@ import ( ) // getCoverage returns the coverage and the percentage of tests that passed -func getCoverage(show bool, getCoverageFunc func() ([]hubtest.Coverage, error)) ([]hubtest.Coverage, int, error) { +func getCoverage(show bool, getCoverageFunc func(string) ([]hubtest.Coverage, error), hubDir string) ([]hubtest.Coverage, int, error) { if !show { return nil, 0, nil } - coverage, err := getCoverageFunc() + coverage, err := getCoverageFunc(hubDir) if err != nil { return nil, 0, fmt.Errorf("while getting coverage: %w", err) } @@ -46,7 +46,7 @@ func (cli *cliHubTest) coverage(showScenarioCov bool, showParserCov bool, showAp // for this one we explicitly don't do for appsec if err := HubTest.LoadAllTests(); err != nil { - return fmt.Errorf("unable to load all tests: %+v", err) + return fmt.Errorf("unable to load all tests: %w", err) } var err error @@ -58,17 +58,17 @@ func (cli *cliHubTest) coverage(showScenarioCov bool, showParserCov bool, showAp showAppsecCov = true } - parserCoverage, parserCoveragePercent, err := getCoverage(showParserCov, HubTest.GetParsersCoverage) + parserCoverage, parserCoveragePercent, err := getCoverage(showParserCov, HubTest.GetParsersCoverage, cfg.Hub.HubDir) if err != nil { return err } - scenarioCoverage, scenarioCoveragePercent, err := getCoverage(showScenarioCov, HubTest.GetScenariosCoverage) + scenarioCoverage, scenarioCoveragePercent, err := getCoverage(showScenarioCov, HubTest.GetScenariosCoverage, cfg.Hub.HubDir) if err != nil { return err } - appsecRuleCoverage, appsecRuleCoveragePercent, err := getCoverage(showAppsecCov, HubTest.GetAppsecCoverage) + appsecRuleCoverage, appsecRuleCoveragePercent, err := getCoverage(showAppsecCov, HubTest.GetAppsecCoverage, cfg.Hub.HubDir) if err != nil { return err } diff --git a/cmd/crowdsec-cli/clihubtest/explain.go b/cmd/crowdsec-cli/clihubtest/explain.go index 6217e44e2..c58122cca 100644 --- a/cmd/crowdsec-cli/clihubtest/explain.go +++ b/cmd/crowdsec-cli/clihubtest/explain.go @@ -1,19 +1,19 @@ package clihubtest import ( + "context" "fmt" "github.com/spf13/cobra" - "github.com/crowdsecurity/crowdsec/pkg/dumps" - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/args" + "github.com/crowdsecurity/crowdsec/pkg/dumps" ) -func (cli *cliHubTest) explain(testName string, details bool, skipOk bool) error { +func (cli *cliHubTest) explain(ctx context.Context, testName string, details bool, skipOk bool) error { test, err := HubTest.LoadTestItem(testName) if err != nil { - return fmt.Errorf("can't load test: %+v", err) + return fmt.Errorf("can't load test: %w", err) } cfg := cli.cfg() @@ -21,8 +21,8 @@ func (cli *cliHubTest) explain(testName string, details bool, skipOk bool) error err = test.ParserAssert.LoadTest(test.ParserResultFile) if err != nil { - if err = test.Run(patternDir); err != nil { - return fmt.Errorf("running test '%s' failed: %+v", test.Name, err) + if err = test.Run(ctx, patternDir); err != nil { + return fmt.Errorf("running test '%s' failed: %w", test.Name, err) } if err = test.ParserAssert.LoadTest(test.ParserResultFile); err != nil { @@ -32,8 +32,8 @@ func (cli *cliHubTest) explain(testName string, details bool, skipOk bool) error err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile) if err != nil { - if err = test.Run(patternDir); err != nil { - return fmt.Errorf("running test '%s' failed: %+v", test.Name, err) + if err = test.Run(ctx, patternDir); err != nil { + return fmt.Errorf("running test '%s' failed: %w", test.Name, err) } if err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile); err != nil { @@ -62,9 +62,10 @@ func (cli *cliHubTest) newExplainCmd() *cobra.Command { Short: "explain [test_name]", Args: args.MinimumNArgs(1), DisableAutoGenTag: true, - RunE: func(_ *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() for _, testName := range args { - if err := cli.explain(testName, details, skipOk); err != nil { + if err := cli.explain(ctx, testName, details, skipOk); err != nil { return err } } diff --git a/cmd/crowdsec-cli/clihubtest/run.go b/cmd/crowdsec-cli/clihubtest/run.go index 94a3b0c10..3d4094cf4 100644 --- a/cmd/crowdsec-cli/clihubtest/run.go +++ b/cmd/crowdsec-cli/clihubtest/run.go @@ -1,34 +1,36 @@ package clihubtest import ( + "context" "encoding/json" "errors" "fmt" - "os" + "runtime" "strings" "github.com/AlecAivazis/survey/v2" "github.com/fatih/color" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" "github.com/crowdsecurity/crowdsec/pkg/emoji" "github.com/crowdsecurity/crowdsec/pkg/hubtest" ) -func (cli *cliHubTest) run(runAll bool, nucleiTargetHost string, appSecHost string, args []string) error { +func (cli *cliHubTest) run(ctx context.Context, all bool, nucleiTargetHost string, appSecHost string, args []string, maxJobs uint) error { cfg := cli.cfg() - if !runAll && len(args) == 0 { + if !all && len(args) == 0 { return errors.New("please provide test to run or --all flag") } hubPtr.NucleiTargetHost = nucleiTargetHost hubPtr.AppSecHost = appSecHost - if runAll { + if all { if err := hubPtr.LoadAllTests(); err != nil { - return fmt.Errorf("unable to load all tests: %+v", err) + return fmt.Errorf("unable to load all tests: %w", err) } } else { for _, testName := range args { @@ -39,23 +41,23 @@ func (cli *cliHubTest) run(runAll bool, nucleiTargetHost string, appSecHost stri } } - // set timezone to avoid DST issues - os.Setenv("TZ", "UTC") - patternDir := cfg.ConfigPaths.PatternDir + var eg errgroup.Group + + eg.SetLimit(int(maxJobs)) + for _, test := range hubPtr.Tests { if cfg.Cscli.Output == "human" { - log.Infof("Running test '%s'", test.Name) + fmt.Printf("Running test '%s'\n", test.Name) } - err := test.Run(patternDir) - if err != nil { - log.Errorf("running test '%s' failed: %+v", test.Name, err) - } + eg.Go(func() error { + return test.Run(ctx, patternDir) + }) } - return nil + return eg.Wait() } func printParserFailures(test *hubtest.HubTestItem) { @@ -101,24 +103,31 @@ func printScenarioFailures(test *hubtest.HubTestItem) { func (cli *cliHubTest) newRunCmd() *cobra.Command { var ( noClean bool - runAll bool + all bool + reportSuccess bool forceClean bool nucleiTargetHost string appSecHost string ) + maxJobs := uint(runtime.NumCPU()) + cmd := &cobra.Command{ Use: "run", Short: "run [test_name]", DisableAutoGenTag: true, - RunE: func(_ *cobra.Command, args []string) error { - return cli.run(runAll, nucleiTargetHost, appSecHost, args) + RunE: func(cmd *cobra.Command, args []string) error { + if all { + fmt.Printf("Running all tests (max_jobs: %d)\n", maxJobs) + } + + return cli.run(cmd.Context(), all, nucleiTargetHost, appSecHost, args, maxJobs) }, PersistentPostRunE: func(_ *cobra.Command, _ []string) error { cfg := cli.cfg() success := true - testResult := make(map[string]bool) + testMap := make(map[string]*hubtest.HubTestItem) for _, test := range hubPtr.Tests { if test.AutoGen && !isAppsecTest { if test.ParserAssert.AutoGenAssert { @@ -132,22 +141,15 @@ func (cli *cliHubTest) newRunCmd() *cobra.Command { fmt.Println(test.ScenarioAssert.AutoGenAssertData) } if !noClean { - if err := test.Clean(); err != nil { - return fmt.Errorf("unable to clean test '%s' env: %w", test.Name, err) - } + test.Clean() } return fmt.Errorf("please fill your assert file(s) for test '%s', exiting", test.Name) } - testResult[test.Name] = test.Success + testMap[test.Name] = test if test.Success { - if cfg.Cscli.Output == "human" { - log.Infof("Test '%s' passed successfully (%d assertions)\n", test.Name, test.ParserAssert.NbAssert+test.ScenarioAssert.NbAssert) - } if !noClean { - if err := test.Clean(); err != nil { - return fmt.Errorf("unable to clean test '%s' env: %w", test.Name, err) - } + test.Clean() } } else { success = false @@ -157,7 +159,7 @@ func (cli *cliHubTest) newRunCmd() *cobra.Command { printScenarioFailures(test) if !forceClean && !noClean { prompt := &survey.Confirm{ - Message: fmt.Sprintf("\nDo you want to remove runtime folder for test '%s'? (default: Yes)", test.Name), + Message: fmt.Sprintf("Do you want to remove runtime and result folder for '%s'?", test.Name), Default: true, } if err := survey.AskOne(prompt, &cleanTestEnv); err != nil { @@ -167,22 +169,20 @@ func (cli *cliHubTest) newRunCmd() *cobra.Command { } if cleanTestEnv || forceClean { - if err := test.Clean(); err != nil { - return fmt.Errorf("unable to clean test '%s' env: %w", test.Name, err) - } + test.Clean() } } } switch cfg.Cscli.Output { case "human": - hubTestResultTable(color.Output, cfg.Cscli.Color, testResult) + hubTestResultTable(color.Output, cfg.Cscli.Color, testMap, reportSuccess) case "json": jsonResult := make(map[string][]string, 0) jsonResult["success"] = make([]string, 0) jsonResult["fail"] = make([]string, 0) - for testName, success := range testResult { - if success { + for testName, test := range testMap { + if test.Success { jsonResult["success"] = append(jsonResult["success"], testName) } else { jsonResult["fail"] = append(jsonResult["fail"], testName) @@ -198,7 +198,11 @@ func (cli *cliHubTest) newRunCmd() *cobra.Command { } if !success { - return errors.New("some tests failed") + if reportSuccess { + return errors.New("some tests failed") + } + + return errors.New("some tests failed, use --report-success to show them all") } return nil @@ -209,7 +213,9 @@ func (cli *cliHubTest) newRunCmd() *cobra.Command { cmd.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail") cmd.Flags().StringVar(&nucleiTargetHost, "target", hubtest.DefaultNucleiTarget, "Target for AppSec Test") cmd.Flags().StringVar(&appSecHost, "host", hubtest.DefaultAppsecHost, "Address to expose AppSec for hubtest") - cmd.Flags().BoolVar(&runAll, "all", false, "Run all tests") + cmd.Flags().BoolVar(&all, "all", false, "Run all tests") + cmd.Flags().BoolVar(&reportSuccess, "report-success", false, "Report successful tests too (implied with json output)") + cmd.Flags().UintVar(&maxJobs, "max-jobs", maxJobs, "Run batch") return cmd } diff --git a/cmd/crowdsec-cli/clihubtest/table.go b/cmd/crowdsec-cli/clihubtest/table.go index 2a105a1f5..b1311a624 100644 --- a/cmd/crowdsec-cli/clihubtest/table.go +++ b/cmd/crowdsec-cli/clihubtest/table.go @@ -3,6 +3,7 @@ package clihubtest import ( "fmt" "io" + "strconv" "github.com/jedib0t/go-pretty/v6/text" @@ -11,22 +12,31 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/hubtest" ) -func hubTestResultTable(out io.Writer, wantColor string, testResult map[string]bool) { +func hubTestResultTable(out io.Writer, wantColor string, testMap map[string]*hubtest.HubTestItem, reportSuccess bool) { t := cstable.NewLight(out, wantColor) - t.SetHeaders("Test", "Result") + t.SetHeaders("Test", "Result", "Assertions") t.SetHeaderAlignment(text.AlignLeft) t.SetAlignment(text.AlignLeft) - for testName, success := range testResult { + showTable := reportSuccess + + for testName, test := range testMap { status := emoji.CheckMarkButton - if !success { + if !test.Success { status = emoji.CrossMark + showTable = true } - t.AddRow(testName, status) + if !test.Success || reportSuccess { + t.AddRow(testName, status, strconv.Itoa(test.ParserAssert.NbAssert+test.ScenarioAssert.NbAssert)) + } } - t.Render() + if showTable { + t.Render() + } else { + fmt.Println("All tests passed, use --report-success for more details.") + } } func hubTestListTable(out io.Writer, wantColor string, tests []*hubtest.HubTestItem) { diff --git a/pkg/dumps/parser_dump.go b/pkg/dumps/parser_dump.go index bd385bec1..64bac2ed7 100644 --- a/pkg/dumps/parser_dump.go +++ b/pkg/dumps/parser_dump.go @@ -35,6 +35,8 @@ type DumpOpts struct { } func LoadParserDump(filepath string) (*ParserResults, error) { + logger := log.WithField("file", filepath) + dumpData, err := os.Open(filepath) if err != nil { return nil, err @@ -83,9 +85,9 @@ func LoadParserDump(filepath string) (*ParserResults, error) { for idx, result := range pdump[lastStage][lastParser] { if result.Evt.StrTime == "" { - log.Warningf("Line %d/%d is missing evt.StrTime. It is most likely a mistake as it will prevent your logs to be processed in time-machine/forensic mode.", idx, len(pdump[lastStage][lastParser])) + logger.Warningf("Line %d/%d is missing evt.StrTime. It is most likely a mistake as it will prevent your logs to be processed in time-machine/forensic mode.", idx, len(pdump[lastStage][lastParser])) } else { - log.Debugf("Line %d/%d has evt.StrTime set to '%s'", idx, len(pdump[lastStage][lastParser]), result.Evt.StrTime) + logger.Debugf("Line %d/%d has evt.StrTime set to '%s'", idx, len(pdump[lastStage][lastParser]), result.Evt.StrTime) } } diff --git a/pkg/hubtest/coverage.go b/pkg/hubtest/coverage.go index e42c1e234..057fc7e70 100644 --- a/pkg/hubtest/coverage.go +++ b/pkg/hubtest/coverage.go @@ -23,7 +23,7 @@ type Coverage struct { PresentIn map[string]bool // poorman's set } -func (h *HubTest) GetAppsecCoverage() ([]Coverage, error) { +func (h *HubTest) GetAppsecCoverage(hubDir string) ([]Coverage, error) { if len(h.HubIndex.GetItemMap(cwhub.APPSEC_RULES)) == 0 { return nil, errors.New("no appsec rules in hub index") } @@ -41,7 +41,7 @@ func (h *HubTest) GetAppsecCoverage() ([]Coverage, error) { } // parser the expressions a-la-oneagain - appsecTestConfigs, err := filepath.Glob(".appsec-tests/*/config.yaml") + appsecTestConfigs, err := filepath.Glob(filepath.Join(hubDir, ".appsec-tests", "*", "config.yaml")) if err != nil { return nil, fmt.Errorf("while find appsec-tests config: %w", err) } @@ -57,7 +57,7 @@ func (h *HubTest) GetAppsecCoverage() ([]Coverage, error) { err = yaml.Unmarshal(yamlFile, configFileData) if err != nil { - return nil, fmt.Errorf("parsing: %v", err) + return nil, fmt.Errorf("parsing: %w", err) } for _, appsecRulesFile := range configFileData.AppsecRules { @@ -70,7 +70,7 @@ func (h *HubTest) GetAppsecCoverage() ([]Coverage, error) { err = yaml.Unmarshal(yamlFile, appsecRuleData) if err != nil { - return nil, fmt.Errorf("parsing: %v", err) + return nil, fmt.Errorf("parsing: %w", err) } appsecRuleName := appsecRuleData.Name @@ -87,7 +87,7 @@ func (h *HubTest) GetAppsecCoverage() ([]Coverage, error) { return coverage, nil } -func (h *HubTest) GetParsersCoverage() ([]Coverage, error) { +func (h *HubTest) GetParsersCoverage(hubDir string) ([]Coverage, error) { if len(h.HubIndex.GetItemMap(cwhub.PARSERS)) == 0 { return nil, errors.New("no parsers in hub index") } @@ -105,7 +105,7 @@ func (h *HubTest) GetParsersCoverage() ([]Coverage, error) { } // parser the expressions a-la-oneagain - passerts, err := filepath.Glob(".tests/*/parser.assert") + passerts, err := filepath.Glob(filepath.Join(hubDir, ".tests", "*", "parser.assert")) if err != nil { return nil, fmt.Errorf("while find parser asserts: %w", err) } @@ -173,7 +173,7 @@ func (h *HubTest) GetParsersCoverage() ([]Coverage, error) { return coverage, nil } -func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) { +func (h *HubTest) GetScenariosCoverage(hubDir string) ([]Coverage, error) { if len(h.HubIndex.GetItemMap(cwhub.SCENARIOS)) == 0 { return nil, errors.New("no scenarios in hub index") } @@ -191,7 +191,7 @@ func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) { } // parser the expressions a-la-oneagain - passerts, err := filepath.Glob(".tests/*/scenario.assert") + passerts, err := filepath.Glob(filepath.Join(hubDir, ".tests", "*", "scenario.assert")) if err != nil { return nil, fmt.Errorf("while find scenario asserts: %w", err) } @@ -259,6 +259,7 @@ func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) { } } } + file.Close() } diff --git a/pkg/hubtest/helpers.go b/pkg/hubtest/helpers.go new file mode 100644 index 000000000..d4714d86f --- /dev/null +++ b/pkg/hubtest/helpers.go @@ -0,0 +1,10 @@ +package hubtest + +import ( + "path/filepath" +) + +func basename(params ...any) (any, error) { + s := params[0].(string) + return filepath.Base(s), nil +} diff --git a/pkg/hubtest/hubtest.go b/pkg/hubtest/hubtest.go index 6e5a11fff..a99d6cc46 100644 --- a/pkg/hubtest/hubtest.go +++ b/pkg/hubtest/hubtest.go @@ -24,13 +24,13 @@ type HubTest struct { TemplateAppsecProfilePath string NucleiTargetHost string AppSecHost string - - HubIndex *cwhub.Hub - Tests []*HubTestItem + DataDir string // we share this one across tests, to avoid unnecessary downloads + HubIndex *cwhub.Hub + Tests []*HubTestItem } const ( - templateConfigFile = "template_config.yaml" + templateConfigFile = "template_config2.yaml" templateSimulationFile = "template_simulation.yaml" templateProfileFile = "template_profiles.yaml" templateAcquisFile = "template_acquis.yaml" @@ -61,7 +61,7 @@ http: func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecTest bool) (HubTest, error) { hubPath, err := filepath.Abs(hubPath) if err != nil { - return HubTest{}, fmt.Errorf("can't get absolute path of hub: %+v", err) + return HubTest{}, fmt.Errorf("can't get absolute path of hub: %w", err) } // we can't use hubtest without the hub @@ -139,9 +139,15 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecT return HubTest{}, err } + dataDir := filepath.Join(hubPath, ".cache", "data") + if err = os.MkdirAll(dataDir, 0o700); err != nil { + return HubTest{}, fmt.Errorf("while creating data dir: %w", err) + } + return HubTest{ CrowdSecPath: crowdsecPath, CscliPath: cscliPath, + DataDir: dataDir, HubPath: hubPath, HubTestPath: HubTestPath, HubIndexFile: hubIndexFile, @@ -155,7 +161,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecT func (h *HubTest) LoadTestItem(name string) (*HubTestItem, error) { HubTestItem := &HubTestItem{} - testItem, err := NewTest(name, h) + testItem, err := NewTest(name, h, h.DataDir) if err != nil { return HubTestItem, err } diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index 75895dc72..be467be02 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -4,11 +4,13 @@ import ( "context" "errors" "fmt" + "io/fs" "net/url" "os" "os/exec" "path/filepath" "strings" + "sync" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" @@ -19,6 +21,8 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/parser" ) +var downloadMutex sync.Mutex + type HubTestItemConfig struct { Parsers []string `yaml:"parsers,omitempty"` Scenarios []string `yaml:"scenarios,omitempty"` @@ -31,6 +35,7 @@ type HubTestItemConfig struct { Labels map[string]string `yaml:"labels,omitempty"` IgnoreParsers bool `yaml:"ignore_parsers,omitempty"` // if we test a scenario, we don't want to assert on Parser OverrideStatics []parser.ExtraField `yaml:"override_statics,omitempty"` // Allow to override statics. Executed before s00 + OwnDataDir bool `yaml:"own_data_dir,omitempty"` // Don't share dataDir with the other tests } type HubTestItem struct { @@ -41,6 +46,7 @@ type HubTestItem struct { CscliPath string RuntimePath string + RuntimeDBDir string RuntimeHubPath string RuntimeDataPath string RuntimePatternsPath string @@ -95,7 +101,7 @@ const ( DefaultAppsecHost = "127.0.0.1:4241" ) -func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { +func NewTest(name string, hubTest *HubTest, dataDir string) (*HubTestItem, error) { testPath := filepath.Join(hubTest.HubTestPath, name) runtimeFolder := filepath.Join(testPath, "runtime") runtimeHubFolder := filepath.Join(runtimeFolder, "hub") @@ -121,6 +127,15 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { scenarioAssertFilePath := filepath.Join(testPath, ScenarioAssertFileName) ScenarioAssert := NewScenarioAssert(scenarioAssertFilePath) + // force own_data_dir for backard compatibility + if name == "magento-ccs-by-as" || name == "magento-ccs-by-country" || name == "geoip-enrich" { + configFileData.OwnDataDir = true + } + + if configFileData.OwnDataDir { + dataDir = filepath.Join(runtimeFolder, "data") + } + return &HubTestItem{ Name: name, Path: testPath, @@ -128,8 +143,8 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { CscliPath: hubTest.CscliPath, RuntimePath: filepath.Join(testPath, "runtime"), RuntimeHubPath: runtimeHubFolder, - RuntimeDataPath: filepath.Join(runtimeFolder, "data"), RuntimePatternsPath: filepath.Join(runtimeFolder, "patterns"), + RuntimeDBDir: filepath.Join(runtimeFolder, "data"), RuntimeConfigFilePath: filepath.Join(runtimeFolder, "config.yaml"), RuntimeProfileFilePath: filepath.Join(runtimeFolder, "profiles.yaml"), RuntimeSimulationFilePath: filepath.Join(runtimeFolder, "simulation.yaml"), @@ -142,7 +157,7 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { HubDir: runtimeHubFolder, HubIndexFile: hubTest.HubIndexFile, InstallDir: runtimeFolder, - InstallDataDir: filepath.Join(runtimeFolder, "data"), + InstallDataDir: dataDir, }, Config: configFileData, HubPath: hubTest.HubPath, @@ -176,7 +191,7 @@ func (t *HubTestItem) installHubItems(names []string, installFunc func(string) e return nil } -func (t *HubTestItem) InstallHub() error { +func (t *HubTestItem) InstallHub(ctx context.Context) error { if err := t.installHubItems(t.Config.Parsers, t.installParser); err != nil { return err } @@ -221,12 +236,14 @@ func (t *HubTestItem) InstallHub() error { return err } - ctx := context.Background() + // prevent concurrent downloads of the same file + downloadMutex.Lock() + defer downloadMutex.Unlock() // install data for parsers if needed for _, item := range hub.GetInstalledByType(cwhub.PARSERS, true) { - if _, err := hubops.DownloadDataIfNeeded(ctx, hub, item, true); err != nil { - return fmt.Errorf("unable to download data for parser '%s': %+v", item.Name, err) + if _, err := hubops.DownloadDataIfNeeded(ctx, hub, item, false); err != nil { + return fmt.Errorf("unable to download data for parser '%s': %w", item.Name, err) } log.Debugf("parser '%s' installed successfully in runtime environment", item.Name) @@ -234,8 +251,8 @@ func (t *HubTestItem) InstallHub() error { // install data for scenarios if needed for _, item := range hub.GetInstalledByType(cwhub.SCENARIOS, true) { - if _, err := hubops.DownloadDataIfNeeded(ctx, hub, item, true); err != nil { - return fmt.Errorf("unable to download data for parser '%s': %+v", item.Name, err) + if _, err := hubops.DownloadDataIfNeeded(ctx, hub, item, false); err != nil { + return fmt.Errorf("unable to download data for parser '%s': %w", item.Name, err) } log.Debugf("scenario '%s' installed successfully in runtime environment", item.Name) @@ -243,8 +260,8 @@ func (t *HubTestItem) InstallHub() error { // install data for postoverflows if needed for _, item := range hub.GetInstalledByType(cwhub.POSTOVERFLOWS, true) { - if _, err := hubops.DownloadDataIfNeeded(ctx, hub, item, true); err != nil { - return fmt.Errorf("unable to download data for parser '%s': %+v", item.Name, err) + if _, err := hubops.DownloadDataIfNeeded(ctx, hub, item, false); err != nil { + return fmt.Errorf("unable to download data for parser '%s': %w", item.Name, err) } log.Debugf("postoverflow '%s' installed successfully in runtime environment", item.Name) @@ -253,49 +270,61 @@ func (t *HubTestItem) InstallHub() error { return nil } -func (t *HubTestItem) Clean() error { - return os.RemoveAll(t.RuntimePath) +func (t *HubTestItem) Clean() { + if err := os.RemoveAll(t.ResultsPath); err != nil { + if !errors.Is(err, fs.ErrNotExist) { + log.Errorf("while cleaning %s: %s", t.Name, err.Error()) + } + } + + if err := os.RemoveAll(t.RuntimePath); err != nil { + if !errors.Is(err, fs.ErrNotExist) { + log.Errorf("while cleaning %s: %s", t.Name, err.Error()) + } + } } func (t *HubTestItem) RunWithNucleiTemplate() error { - crowdsecLogFile := fmt.Sprintf("%s/log/crowdsec.log", t.RuntimePath) - testPath := filepath.Join(t.HubTestPath, t.Name) if _, err := os.Stat(testPath); os.IsNotExist(err) { return fmt.Errorf("test '%s' doesn't exist in '%s', exiting", t.Name, t.HubTestPath) } - if err := os.Chdir(testPath); err != nil { - return fmt.Errorf("can't 'cd' to '%s': %w", testPath, err) - } + crowdsecLogFile := fmt.Sprintf("%s/log/crowdsec.log", t.RuntimePath) // machine add cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--force", "--auto"} cscliRegisterCmd := exec.Command(t.CscliPath, cmdArgs...) + cscliRegisterCmd.Dir = testPath + cscliRegisterCmd.Env = []string{"TESTDIR="+testPath, "DATADIR="+t.RuntimeHubConfig.InstallDataDir, "TZ=UTC"} output, err := cscliRegisterCmd.CombinedOutput() if err != nil { if !strings.Contains(string(output), "unable to create machine: user 'testMachine': user already exist") { fmt.Println(string(output)) - return fmt.Errorf("fail to run '%s' for test '%s': %v", cscliRegisterCmd.String(), t.Name, err) + return fmt.Errorf("fail to run '%s' for test '%s': %w", cscliRegisterCmd.String(), t.Name, err) } } // hardcode bouncer key cmdArgs = []string{"-c", t.RuntimeConfigFilePath, "bouncers", "add", "appsectests", "-k", TestBouncerApiKey} cscliBouncerCmd := exec.Command(t.CscliPath, cmdArgs...) + cscliBouncerCmd.Dir = testPath + cscliBouncerCmd.Env = []string{"TESTDIR="+testPath, "DATADIR="+t.RuntimeHubConfig.InstallDataDir, "TZ=UTC"} output, err = cscliBouncerCmd.CombinedOutput() if err != nil { if !strings.Contains(string(output), "unable to create bouncer: bouncer appsectests already exists") { fmt.Println(string(output)) - return fmt.Errorf("fail to run '%s' for test '%s': %v", cscliRegisterCmd.String(), t.Name, err) + return fmt.Errorf("fail to run '%s' for test '%s': %w", cscliRegisterCmd.String(), t.Name, err) } } // start crowdsec service cmdArgs = []string{"-c", t.RuntimeConfigFilePath} crowdsecDaemon := exec.Command(t.CrowdSecPath, cmdArgs...) + crowdsecDaemon.Dir = testPath + crowdsecDaemon.Env = []string{"TESTDIR="+testPath, "DATADIR="+t.RuntimeHubConfig.InstallDataDir, "TZ=UTC"} crowdsecDaemon.Start() @@ -382,59 +411,16 @@ func createDirs(dirs []string) error { return nil } -func (t *HubTestItem) RunWithLogFile(patternDir string) error { +func (t *HubTestItem) RunWithLogFile() error { testPath := filepath.Join(t.HubTestPath, t.Name) if _, err := os.Stat(testPath); os.IsNotExist(err) { return fmt.Errorf("test '%s' doesn't exist in '%s', exiting", t.Name, t.HubTestPath) } - currentDir, err := os.Getwd() // xx - if err != nil { - return fmt.Errorf("can't get current directory: %+v", err) - } - - // create runtime, data, hub folders - if err = createDirs([]string{t.RuntimePath, t.RuntimeDataPath, t.RuntimeHubPath, t.ResultsPath}); err != nil { - return err - } - - if err = Copy(t.HubIndexFile, filepath.Join(t.RuntimeHubPath, ".index.json")); err != nil { - return fmt.Errorf("unable to copy .index.json file in '%s': %w", filepath.Join(t.RuntimeHubPath, ".index.json"), err) - } - - // copy template config file to runtime folder - if err = Copy(t.TemplateConfigPath, t.RuntimeConfigFilePath); err != nil { - return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateConfigPath, t.RuntimeConfigFilePath, err) - } - - // copy template profile file to runtime folder - if err = Copy(t.TemplateProfilePath, t.RuntimeProfileFilePath); err != nil { - return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateProfilePath, t.RuntimeProfileFilePath, err) - } - - // copy template simulation file to runtime folder - if err = Copy(t.TemplateSimulationPath, t.RuntimeSimulationFilePath); err != nil { - return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateSimulationPath, t.RuntimeSimulationFilePath, err) - } - - // copy template patterns folder to runtime folder - if err = CopyDir(patternDir, t.RuntimePatternsPath); err != nil { - return fmt.Errorf("unable to copy 'patterns' from '%s' to '%s': %w", patternDir, t.RuntimePatternsPath, err) - } - - // install the hub in the runtime folder - if err = t.InstallHub(); err != nil { - return fmt.Errorf("unable to install hub in '%s': %w", t.RuntimeHubPath, err) - } - - logFile := t.Config.LogFile + logFile := filepath.Join(testPath, t.Config.LogFile) logType := t.Config.LogType dsn := fmt.Sprintf("file://%s", logFile) - if err = os.Chdir(testPath); err != nil { - return fmt.Errorf("can't 'cd' to '%s': %w", testPath, err) - } - logFileStat, err := os.Stat(logFile) if err != nil { return fmt.Errorf("unable to stat log file '%s': %w", logFile, err) @@ -446,6 +432,9 @@ func (t *HubTestItem) RunWithLogFile(patternDir string) error { cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--force", "--auto"} cscliRegisterCmd := exec.Command(t.CscliPath, cmdArgs...) + cscliRegisterCmd.Dir = testPath + cscliRegisterCmd.Env = []string{"TESTDIR="+testPath, "DATADIR="+t.RuntimeHubConfig.InstallDataDir, "TZ=UTC"} + log.Debugf("%s", cscliRegisterCmd.String()) output, err := cscliRegisterCmd.CombinedOutput() @@ -464,6 +453,9 @@ func (t *HubTestItem) RunWithLogFile(patternDir string) error { } crowdsecCmd := exec.Command(t.CrowdSecPath, cmdArgs...) + crowdsecCmd.Dir = testPath + crowdsecCmd.Env = []string{"TESTDIR="+testPath, "DATADIR="+t.RuntimeHubConfig.InstallDataDir, "TZ=UTC"} + log.Debugf("%s", crowdsecCmd.String()) output, err = crowdsecCmd.CombinedOutput() @@ -475,10 +467,6 @@ func (t *HubTestItem) RunWithLogFile(patternDir string) error { return fmt.Errorf("fail to run '%s' for test '%s': %v", crowdsecCmd.String(), t.Name, err) } - if err := os.Chdir(currentDir); err != nil { - return fmt.Errorf("can't 'cd' to '%s': %w", currentDir, err) - } - // assert parsers if !t.Config.IgnoreParsers { _, err := os.Stat(t.ParserAssert.File) @@ -506,6 +494,7 @@ func (t *HubTestItem) RunWithLogFile(patternDir string) error { t.ParserAssert.AutoGenAssert = true } else { if err := t.ParserAssert.AssertFile(t.ParserResultFile); err != nil { + // TODO: no error - should not prevent running the other tests return fmt.Errorf("unable to run assertion on file '%s': %w", t.ParserResultFile, err) } } @@ -564,14 +553,14 @@ func (t *HubTestItem) RunWithLogFile(patternDir string) error { return nil } -func (t *HubTestItem) Run(patternDir string) error { +func (t *HubTestItem) Run(ctx context.Context, patternDir string) error { var err error t.Success = false t.ErrorsList = make([]string, 0) // create runtime, data, hub, result folders - if err = createDirs([]string{t.RuntimePath, t.RuntimeDataPath, t.RuntimeHubPath, t.ResultsPath}); err != nil { + if err = createDirs([]string{t.RuntimePath, t.RuntimeDBDir, t.RuntimeHubConfig.InstallDataDir, t.RuntimeHubPath, t.ResultsPath}); err != nil { return err } @@ -625,12 +614,12 @@ func (t *HubTestItem) Run(patternDir string) error { } // install the hub in the runtime folder - if err = t.InstallHub(); err != nil { + if err = t.InstallHub(ctx); err != nil { return fmt.Errorf("unable to install hub in '%s': %w", t.RuntimeHubPath, err) } if t.Config.LogFile != "" { - return t.RunWithLogFile(patternDir) + return t.RunWithLogFile() } if t.Config.NucleiTemplate != "" { diff --git a/pkg/hubtest/parser_assert.go b/pkg/hubtest/parser_assert.go index 90d952506..1e7c7b2b3 100644 --- a/pkg/hubtest/parser_assert.go +++ b/pkg/hubtest/parser_assert.go @@ -61,7 +61,7 @@ func (p *ParserAssert) AutoGenFromFile(filename string) (string, error) { func (p *ParserAssert) LoadTest(filename string) error { parserDump, err := dumps.LoadParserDump(filename) if err != nil { - return fmt.Errorf("loading parser dump file: %+v", err) + return fmt.Errorf("loading parser dump file: %w", err) } p.TestData = parserDump @@ -93,7 +93,7 @@ func (p *ParserAssert) AssertFile(testFile string) error { ok, err := p.Run(scanner.Text()) if err != nil { - return fmt.Errorf("unable to run assert '%s': %+v", scanner.Text(), err) + return fmt.Errorf("unable to run assert '%s': %w", scanner.Text(), err) } p.NbAssert++ @@ -151,26 +151,43 @@ func (p *ParserAssert) AssertFile(testFile string) error { return nil } +func basenameShim(expression string) string { + if strings.Contains(expression, "datasource_path") && !strings.Contains(expression, "basename(") { + // match everything before == and wrap it with basename() + match := strings.Split(expression, "==") + return fmt.Sprintf("basename(%s) == %s", match[0], match[1]) + } + + return expression +} + func (p *ParserAssert) RunExpression(expression string) (interface{}, error) { // debug doesn't make much sense with the ability to evaluate "on the fly" // var debugFilter *exprhelpers.ExprDebugger - var output interface{} + var output any - env := map[string]interface{}{"results": *p.TestData} + logger := log.WithField("file", p.File) - runtimeFilter, err := expr.Compile(expression, exprhelpers.GetExprOptions(env)...) + env := map[string]any{"results": *p.TestData} + opts := exprhelpers.GetExprOptions(env) + opts = append(opts, expr.Function("basename", basename, new(func (string) string))) + + // wrap with basename() in case of datasource_path, for backward compatibility + expression = basenameShim(expression) + + runtimeFilter, err := expr.Compile(expression, opts...) if err != nil { - log.Errorf("failed to compile '%s' : %s", expression, err) + logger.Errorf("failed to compile '%s': %s", expression, err) return output, err } // dump opcode in trace level - log.Tracef("%s", runtimeFilter.Disassemble()) + logger.Tracef("%s", runtimeFilter.Disassemble()) output, err = expr.Run(runtimeFilter, env) if err != nil { - log.Warningf("running : %s", expression) - log.Warningf("runtime error : %s", err) + logger.Warningf("running : %s", expression) + logger.Warningf("runtime error: %s", err) return output, fmt.Errorf("while running expression %s: %w", expression, err) } @@ -252,7 +269,11 @@ func (p *ParserAssert) AutoGenParserAssert() string { continue } - ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Meta["%s"] == "%s"`+"\n", stage, parser, pidx, mkey, Escape(mval)) + if mkey == "datasource_path" { + ret += fmt.Sprintf(`basename(results["%s"]["%s"][%d].Evt.Meta["%s"]) == "%s"`+"\n", stage, parser, pidx, mkey, Escape(mval)) + } else { + ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Meta["%s"] == "%s"`+"\n", stage, parser, pidx, mkey, Escape(mval)) + } } for _, ekey := range maptools.SortedKeys(result.Evt.Enriched) { diff --git a/pkg/hubtest/scenario_assert.go b/pkg/hubtest/scenario_assert.go index f32abf9e1..906751f49 100644 --- a/pkg/hubtest/scenario_assert.go +++ b/pkg/hubtest/scenario_assert.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "path/filepath" "sort" "strings" @@ -59,7 +60,7 @@ func (s *ScenarioAssert) AutoGenFromFile(filename string) (string, error) { func (s *ScenarioAssert) LoadTest(filename string, bucketpour string) error { bucketDump, err := LoadScenarioDump(filename) if err != nil { - return fmt.Errorf("loading scenario dump file '%s': %+v", filename, err) + return fmt.Errorf("loading scenario dump file '%s': %w", filename, err) } s.TestData = bucketDump @@ -67,7 +68,7 @@ func (s *ScenarioAssert) LoadTest(filename string, bucketpour string) error { if bucketpour != "" { pourDump, err := dumps.LoadBucketPourDump(bucketpour) if err != nil { - return fmt.Errorf("loading bucket pour dump file '%s': %+v", filename, err) + return fmt.Errorf("loading bucket pour dump file '%s': %w", filename, err) } s.PourData = pourDump @@ -100,7 +101,7 @@ func (s *ScenarioAssert) AssertFile(testFile string) error { ok, err := s.Run(scanner.Text()) if err != nil { - return fmt.Errorf("unable to run assert '%s': %+v", scanner.Text(), err) + return fmt.Errorf("unable to run assert '%s': %w", scanner.Text(), err) } s.NbAssert++ @@ -156,28 +157,34 @@ func (s *ScenarioAssert) AssertFile(testFile string) error { return nil } -func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) { +func (s *ScenarioAssert) RunExpression(expression string) (any, error) { // debug doesn't make much sense with the ability to evaluate "on the fly" // var debugFilter *exprhelpers.ExprDebugger - var output interface{} + var output any - env := map[string]interface{}{"results": *s.TestData} + logger := log.WithField("file", s.File) - runtimeFilter, err := expr.Compile(expression, exprhelpers.GetExprOptions(env)...) + env := map[string]any{"results": *s.TestData} + opts := exprhelpers.GetExprOptions(env) + opts = append(opts, expr.Function("basename", basename, new(func (string) string))) + + expression = basenameShim(expression) + + runtimeFilter, err := expr.Compile(expression, opts...) if err != nil { return nil, err } // if debugFilter, err = exprhelpers.NewDebugger(assert, expr.Env(env)); err != nil { - // log.Warningf("Failed building debugher for %s : %s", assert, err) + // logger.Warningf("Failed building debugher for %s : %s", assert, err) // } // dump opcode in trace level - log.Tracef("%s", runtimeFilter.Disassemble()) + logger.Tracef("%s", runtimeFilter.Disassemble()) - output, err = expr.Run(runtimeFilter, map[string]interface{}{"results": *s.TestData}) + output, err = expr.Run(runtimeFilter, map[string]any{"results": *s.TestData}) if err != nil { - log.Warningf("running : %s", expression) - log.Warningf("runtime error : %s", err) + logger.Warningf("running : %s", expression) + logger.Warningf("runtime error : %s", err) return nil, fmt.Errorf("while running expression %s: %w", expression, err) } @@ -228,7 +235,11 @@ func (s *ScenarioAssert) AutoGenScenarioAssert() string { for evtIndex, evt := range event.Overflow.Alert.Events { for _, meta := range evt.Meta { - ret += fmt.Sprintf(`results[%d].Overflow.Alert.Events[%d].GetMeta("%s") == "%s"`+"\n", eventIndex, evtIndex, meta.Key, Escape(meta.Value)) + if meta.Key == "datasource_path" { + ret += fmt.Sprintf(`basename(results[%d].Overflow.Alert.Events[%d].GetMeta("%s")) == "%s"`+"\n", eventIndex, evtIndex, meta.Key, Escape(filepath.Base(meta.Value))) + } else { + ret += fmt.Sprintf(`results[%d].Overflow.Alert.Events[%d].GetMeta("%s") == "%s"`+"\n", eventIndex, evtIndex, meta.Key, Escape(meta.Value)) + } } } diff --git a/test/bats.mk b/test/bats.mk index 72ac8863f..7d05d2450 100644 --- a/test/bats.mk +++ b/test/bats.mk @@ -103,7 +103,6 @@ bats-test: bats-environment ## Run functional tests $(TEST_DIR)/run-tests $(TEST_DIR)/bats bats-test-hub: bats-environment bats-check-requirements ## Run all hub tests - @$(TEST_DIR)/bin/generate-hub-tests $(TEST_DIR)/run-tests $(TEST_DIR)/dyn-bats # Not failproof but they can catch bugs and improve learning of sh/bash diff --git a/test/bin/collect-hub-coverage b/test/bin/collect-hub-coverage index 05c4e0625..1072ed867 100755 --- a/test/bin/collect-hub-coverage +++ b/test/bin/collect-hub-coverage @@ -12,13 +12,16 @@ THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) # shellcheck disable=SC1091 . "${THIS_DIR}/../.environment.sh" -hubdir="${LOCAL_DIR}/hub-tests" - coverage() { "${CSCLI}" --crowdsec "${CROWDSEC}" --cscli "${CSCLI}" hubtest coverage --"$1" --percent } -cd "${hubdir}" || die "Could not find hub test results" +hubdir="${LOCAL_DIR}/hub-tests" + +hubdir="${1:-${hubdir}}" + +[[ -d "${hubdir}" ]] || die "Could not find hub test results in $hubdir" +cd "${hubdir}" || die "Could not find hub test results in $hubdir" shopt -s inherit_errexit diff --git a/test/bin/generate-hub-tests b/test/bin/generate-hub-tests deleted file mode 100755 index 658cc33a7..000000000 --- a/test/bin/generate-hub-tests +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -set -eu - -# shellcheck disable=SC1007 -THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -# shellcheck disable=SC1091 -. "${THIS_DIR}/../.environment.sh" - -"${TEST_DIR}/instance-data" load - -hubdir="${LOCAL_DIR}/hub-tests" -git clone --depth 1 https://github.com/crowdsecurity/hub.git "${hubdir}" >/dev/null 2>&1 || (cd "${hubdir}"; git pull) - -echo "Generating hub tests..." - -python3 "$THIS_DIR/generate-hub-tests.py" \ - <("${CSCLI}" --crowdsec "${CROWDSEC}" --cscli "${CSCLI}" hubtest --hub "${hubdir}" list -o json) \ - "${TEST_DIR}/dyn-bats/" diff --git a/test/bin/generate-hub-tests.py b/test/bin/generate-hub-tests.py deleted file mode 100644 index 48f296776..000000000 --- a/test/bin/generate-hub-tests.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python3 - -import json -import pathlib -import os -import sys -import textwrap - -test_header = """ -set -u - -setup_file() { - load "../lib/setup_file.sh" -} - -teardown_file() { - load "../lib/teardown_file.sh" -} - -setup() { - load "../lib/setup.sh" -} -""" - - -def write_chunk(target_dir, n, chunk): - with open(target_dir / f"hub-{n}.bats", "w") as f: - f.write(test_header) - for test in chunk: - cscli = os.environ['CSCLI'] - crowdsec = os.environ['CROWDSEC'] - testname = test['Name'] - hubdir = os.environ['LOCAL_DIR'] + '/hub-tests' - f.write(textwrap.dedent(f""" - @test "{testname}" {{ - run "{cscli}" \\ - --crowdsec "{crowdsec}" \\ - --cscli "{cscli}" \\ - --hub "{hubdir}" \\ - hubtest run "{testname}" \\ - --clean - echo "$output" - assert_success - }} - """)) - - -def main(): - hubtests_json = sys.argv[1] - target_dir = sys.argv[2] - - with open(hubtests_json) as f: - j = json.load(f) - chunk_size = len(j) // 3 + 1 - n = 1 - for i in range(0, len(j), chunk_size): - chunk = j[i:i + chunk_size] - write_chunk(pathlib.Path(target_dir), n, chunk) - n += 1 - - -if __name__ == "__main__": - main()