diff --git a/Makefile b/Makefile index 6bd3cbb79..a0b06dc2e 100644 --- a/Makefile +++ b/Makefile @@ -138,7 +138,8 @@ COMPONENTS := \ datasource_loki \ datasource_s3 \ datasource_syslog \ - datasource_wineventlog + datasource_wineventlog \ + cscli_setup comma := , space := $(empty) $(empty) diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index 01179cf93..1cca03b1d 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -28,7 +28,6 @@ import ( "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/climetrics" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/clinotifications" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/clipapi" - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/clisetup" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/clisimulation" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/clisupport" "github.com/crowdsecurity/crowdsec/pkg/csconfig" @@ -281,9 +280,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall cmd.AddCommand(cliitem.NewAppsecConfig(cli.cfg).NewCommand()) cmd.AddCommand(cliitem.NewAppsecRule(cli.cfg).NewCommand()) - if fflag.CscliSetup.IsEnabled() { - cmd.AddCommand(clisetup.New(cli.cfg).NewCommand()) - } + cli.addSetup(cmd) if len(os.Args) > 1 { cobra.OnInitialize( diff --git a/cmd/crowdsec-cli/setup.go b/cmd/crowdsec-cli/setup.go new file mode 100644 index 000000000..66c0d71e7 --- /dev/null +++ b/cmd/crowdsec-cli/setup.go @@ -0,0 +1,18 @@ +//go:build !no_cscli_setup +package main + +import ( + "github.com/spf13/cobra" + + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/clisetup" + "github.com/crowdsecurity/crowdsec/pkg/cwversion/component" + "github.com/crowdsecurity/crowdsec/pkg/fflag" +) + +func (cli *cliRoot) addSetup(cmd *cobra.Command) { + if fflag.CscliSetup.IsEnabled() { + cmd.AddCommand(clisetup.New(cli.cfg).NewCommand()) + } + + component.Register("cscli_setup") +} diff --git a/cmd/crowdsec-cli/setup_stub.go b/cmd/crowdsec-cli/setup_stub.go new file mode 100644 index 000000000..e001f93c7 --- /dev/null +++ b/cmd/crowdsec-cli/setup_stub.go @@ -0,0 +1,9 @@ +//go:build no_cscli_setup +package main + +import ( + "github.com/spf13/cobra" +) + +func (cli *cliRoot) addSetup(_ *cobra.Command) { +} diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index b2493bbb9..4a5226a29 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -19,6 +19,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/cwversion/component" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -54,44 +55,34 @@ type DataSource interface { var ( // We declare everything here so we can tell if they are unsupported, or excluded from the build - AcquisitionSources = map[string]func() DataSource{ - "appsec": nil, - "cloudwatch": nil, - "docker": nil, - "file": nil, - "journalctl": nil, - "k8s-audit": nil, - "kafka": nil, - "kinesis": nil, - "loki": nil, - "s3": nil, - "syslog": nil, - "wineventlog": nil, - } - transformRuntimes = map[string]*vm.Program{} + AcquisitionSources = map[string]func() DataSource{} + transformRuntimes = map[string]*vm.Program{} ) func GetDataSourceIface(dataSourceType string) (DataSource, error) { - source, ok := AcquisitionSources[dataSourceType] - if !ok { + source, registered := AcquisitionSources[dataSourceType] + if registered { + return source(), nil + } + + built, known := component.Built["datasource_"+dataSourceType] + + if !known { return nil, fmt.Errorf("unknown data source %s", dataSourceType) } - if source == nil { - return nil, fmt.Errorf("data source %s is not built in this version of crowdsec", dataSourceType) + if built { + panic("datasource " + dataSourceType + " is built but not registered") } - return source(), nil + return nil, fmt.Errorf("data source %s is not built in this version of crowdsec", dataSourceType) } // registerDataSource registers a datasource in the AcquisitionSources map. // It must be called in the init() function of the datasource package, and the datasource name // must be declared with a nil value in the map, to allow for conditional compilation. func registerDataSource(dataSourceType string, dsGetter func() DataSource) { - _, ok := AcquisitionSources[dataSourceType] - if !ok { - panic("datasource must be declared in the map: " + dataSourceType) - } + component.Register("datasource_" + dataSourceType) AcquisitionSources[dataSourceType] = dsGetter } @@ -214,9 +205,11 @@ func GetMetricsLevelFromPromCfg(prom *csconfig.PrometheusCfg) int { if prom == nil { return configuration.METRICS_FULL } + if !prom.Enabled { return configuration.METRICS_NONE } + if prom.Level == configuration.CFG_METRICS_AGGREGATE { return configuration.METRICS_AGGREGATE } @@ -224,6 +217,7 @@ func GetMetricsLevelFromPromCfg(prom *csconfig.PrometheusCfg) int { if prom.Level == configuration.CFG_METRICS_FULL { return configuration.METRICS_FULL } + return configuration.METRICS_FULL } @@ -232,15 +226,20 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg, prom *csconfig var sources []DataSource metrics_level := GetMetricsLevelFromPromCfg(prom) + for _, acquisFile := range config.AcquisitionFiles { log.Infof("loading acquisition file : %s", acquisFile) + yamlFile, err := os.Open(acquisFile) if err != nil { return nil, err } + dec := yaml.NewDecoder(yamlFile) dec.SetStrict(true) + idx := -1 + for { var sub configuration.DataSourceCommonCfg err = dec.Decode(&sub) @@ -249,7 +248,9 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg, prom *csconfig if !errors.Is(err, io.EOF) { return nil, fmt.Errorf("failed to yaml decode %s: %w", acquisFile, err) } + log.Tracef("End of yaml file") + break } @@ -263,11 +264,13 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg, prom *csconfig log.Debugf("skipping empty item in %s", acquisFile) continue } + if sub.Source != "docker" { // docker is the only source that can be empty return nil, fmt.Errorf("missing labels in %s (position: %d)", acquisFile, idx) } } + if sub.Source == "" { return nil, fmt.Errorf("data source type is empty ('source') in %s (position: %d)", acquisFile, idx) } diff --git a/pkg/cwversion/component/component.go b/pkg/cwversion/component/component.go new file mode 100644 index 000000000..4036b63cf --- /dev/null +++ b/pkg/cwversion/component/component.go @@ -0,0 +1,34 @@ +package component + +// Package component provides functionality for managing the registration of +// optional, compile-time components in the system. This is meant as a space +// saving measure, separate from feature flags (package pkg/fflag) which are +// only enabled/disabled at runtime. + +// Built is a map of all the known components, and whether they are built-in or not. +// This is populated as soon as possible by the respective init() functions +var Built = map[string]bool { + "datasource_appsec": false, + "datasource_cloudwatch": false, + "datasource_docker": false, + "datasource_file": false, + "datasource_journalctl": false, + "datasource_k8s-audit": false, + "datasource_kafka": false, + "datasource_kinesis": false, + "datasource_loki": false, + "datasource_s3": false, + "datasource_syslog": false, + "datasource_wineventlog":false, + "cscli_setup": false, +} + +func Register(name string) { + if _, ok := Built[name]; !ok { + // having a list of the disabled components is essential + // to debug users' issues + panic("cannot register unknown compile-time component: " + name) + } + + Built[name] = true +} diff --git a/pkg/cwversion/version.go b/pkg/cwversion/version.go index 867098e7d..2cb7de13e 100644 --- a/pkg/cwversion/version.go +++ b/pkg/cwversion/version.go @@ -7,8 +7,8 @@ import ( "github.com/crowdsecurity/go-cs-lib/maptools" "github.com/crowdsecurity/go-cs-lib/version" - "github.com/crowdsecurity/crowdsec/pkg/acquisition" "github.com/crowdsecurity/crowdsec/pkg/apiclient/useragent" + "github.com/crowdsecurity/crowdsec/pkg/cwversion/component" "github.com/crowdsecurity/crowdsec/pkg/cwversion/constraint" ) @@ -18,16 +18,16 @@ var ( ) func FullString() string { - dsBuilt := []string{} - dsExcluded := []string{} + dsBuilt := map[string]struct{}{} + dsExcluded := map[string]struct{}{} - for _, ds := range maptools.SortedKeys(acquisition.AcquisitionSources) { - if acquisition.AcquisitionSources[ds] != nil { - dsBuilt = append(dsBuilt, ds) + for ds, built := range component.Built { + if built { + dsBuilt[ds] = struct{}{} continue } - dsExcluded = append(dsExcluded, ds) + dsExcluded[ds] = struct{}{} } ret := fmt.Sprintf("version: %s\n", version.String()) @@ -42,12 +42,16 @@ func FullString() string { ret += fmt.Sprintf("Constraint_api: %s\n", constraint.API) ret += fmt.Sprintf("Constraint_acquis: %s\n", constraint.Acquis) + built := "(none)" + if len(dsBuilt) > 0 { - ret += fmt.Sprintf("Built data sources: %s\n", strings.Join(dsBuilt, ", ")) + built = strings.Join(maptools.SortedKeys(dsBuilt), ", ") } + ret += fmt.Sprintf("Built-in optional components: %s\n", built) + if len(dsExcluded) > 0 { - ret += fmt.Sprintf("Excluded data sources: %s\n", strings.Join(dsExcluded, ", ")) + ret += fmt.Sprintf("Excluded components: %s\n", strings.Join(maptools.SortedKeys(dsExcluded), ", ")) } return ret