remove dependency from pkg/cwversion to pkg/acquisition (#3242)

* register built-in components without dependencies
* package comment

Makefile

@@ -138,7 +138,8 @@ COMPONENTS := \
 	datasource_loki \
 	datasource_s3 \
 	datasource_syslog \
-	datasource_wineventlog
+	datasource_wineventlog \
+	cscli_setup
 
 comma := ,
 space := $(empty) $(empty)

cmd/crowdsec-cli/main.go

@@ -28,7 +28,6 @@ import (
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/climetrics"
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/clinotifications"
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/clipapi"
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/clisetup"
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/clisimulation"
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/clisupport"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
@@ -281,9 +280,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
 	cmd.AddCommand(cliitem.NewAppsecConfig(cli.cfg).NewCommand())
 	cmd.AddCommand(cliitem.NewAppsecRule(cli.cfg).NewCommand())
 
-	if fflag.CscliSetup.IsEnabled() {
-		cmd.AddCommand(clisetup.New(cli.cfg).NewCommand())
-	}
+	cli.addSetup(cmd)
 
 	if len(os.Args) > 1 {
 		cobra.OnInitialize(

cmd/crowdsec-cli/setup.go

@@ -0,0 +1,18 @@
+//go:build !no_cscli_setup
+package main
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/clisetup"
+	"github.com/crowdsecurity/crowdsec/pkg/cwversion/component"
+	"github.com/crowdsecurity/crowdsec/pkg/fflag"
+)
+
+func (cli *cliRoot) addSetup(cmd *cobra.Command) {
+	if fflag.CscliSetup.IsEnabled() {
+		cmd.AddCommand(clisetup.New(cli.cfg).NewCommand())
+	}
+
+	component.Register("cscli_setup")
+}

cmd/crowdsec-cli/setup_stub.go

@@ -0,0 +1,9 @@
+//go:build no_cscli_setup
+package main
+
+import (
+	"github.com/spf13/cobra"
+)
+
+func (cli *cliRoot) addSetup(_ *cobra.Command) {
+}

pkg/acquisition/acquisition.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/crowdsecurity/crowdsec/pkg/cwversion/component"
 	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
@@ -54,44 +55,34 @@ type DataSource interface {
 
 var (
 	// We declare everything here so we can tell if they are unsupported, or excluded from the build
-	AcquisitionSources = map[string]func() DataSource{
-		"appsec":      nil,
-		"cloudwatch":  nil,
-		"docker":      nil,
-		"file":        nil,
-		"journalctl":  nil,
-		"k8s-audit":   nil,
-		"kafka":       nil,
-		"kinesis":     nil,
-		"loki":        nil,
-		"s3":          nil,
-		"syslog":      nil,
-		"wineventlog": nil,
-	}
-	transformRuntimes = map[string]*vm.Program{}
+	AcquisitionSources = map[string]func() DataSource{}
+	transformRuntimes  = map[string]*vm.Program{}
 )
 
 func GetDataSourceIface(dataSourceType string) (DataSource, error) {
-	source, ok := AcquisitionSources[dataSourceType]
-	if !ok {
+	source, registered := AcquisitionSources[dataSourceType]
+	if registered {
+		return source(), nil
+	}
+
+	built, known := component.Built["datasource_"+dataSourceType]
+
+	if !known {
 		return nil, fmt.Errorf("unknown data source %s", dataSourceType)
 	}
 
-	if source == nil {
-		return nil, fmt.Errorf("data source %s is not built in this version of crowdsec", dataSourceType)
+	if built {
+		panic("datasource " + dataSourceType + " is built but not registered")
 	}
 
-	return source(), nil
+	return nil, fmt.Errorf("data source %s is not built in this version of crowdsec", dataSourceType)
 }
 
 // registerDataSource registers a datasource in the AcquisitionSources map.
 // It must be called in the init() function of the datasource package, and the datasource name
 // must be declared with a nil value in the map, to allow for conditional compilation.
 func registerDataSource(dataSourceType string, dsGetter func() DataSource) {
-	_, ok := AcquisitionSources[dataSourceType]
-	if !ok {
-		panic("datasource must be declared in the map: " + dataSourceType)
-	}
+	component.Register("datasource_" + dataSourceType)
 
 	AcquisitionSources[dataSourceType] = dsGetter
 }
@@ -214,9 +205,11 @@ func GetMetricsLevelFromPromCfg(prom *csconfig.PrometheusCfg) int {
 	if prom == nil {
 		return configuration.METRICS_FULL
 	}
+
 	if !prom.Enabled {
 		return configuration.METRICS_NONE
 	}
+
 	if prom.Level == configuration.CFG_METRICS_AGGREGATE {
 		return configuration.METRICS_AGGREGATE
 	}
@@ -224,6 +217,7 @@ func GetMetricsLevelFromPromCfg(prom *csconfig.PrometheusCfg) int {
 	if prom.Level == configuration.CFG_METRICS_FULL {
 		return configuration.METRICS_FULL
 	}
+
 	return configuration.METRICS_FULL
 }
 
@@ -232,15 +226,20 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg, prom *csconfig
 	var sources []DataSource
 
 	metrics_level := GetMetricsLevelFromPromCfg(prom)
+
 	for _, acquisFile := range config.AcquisitionFiles {
 		log.Infof("loading acquisition file : %s", acquisFile)
+
 		yamlFile, err := os.Open(acquisFile)
 		if err != nil {
 			return nil, err
 		}
+
 		dec := yaml.NewDecoder(yamlFile)
 		dec.SetStrict(true)
+
 		idx := -1
+
 		for {
 			var sub configuration.DataSourceCommonCfg
 			err = dec.Decode(&sub)
@@ -249,7 +248,9 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg, prom *csconfig
 				if !errors.Is(err, io.EOF) {
 					return nil, fmt.Errorf("failed to yaml decode %s: %w", acquisFile, err)
 				}
+
 				log.Tracef("End of yaml file")
+
 				break
 			}
 
@@ -263,11 +264,13 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg, prom *csconfig
 					log.Debugf("skipping empty item in %s", acquisFile)
 					continue
 				}
+
 				if sub.Source != "docker" {
 					// docker is the only source that can be empty
 					return nil, fmt.Errorf("missing labels in %s (position: %d)", acquisFile, idx)
 				}
 			}
+
 			if sub.Source == "" {
 				return nil, fmt.Errorf("data source type is empty ('source') in %s (position: %d)", acquisFile, idx)
 			}

pkg/cwversion/component/component.go

@@ -0,0 +1,34 @@
+package component
+
+// Package component provides functionality for managing the registration of
+// optional, compile-time components in the system. This is meant as a space
+// saving measure, separate from feature flags (package pkg/fflag) which are
+// only enabled/disabled at runtime.
+
+// Built is a map of all the known components, and whether they are built-in or not.
+// This is populated as soon as possible by the respective init() functions
+var Built = map[string]bool {
+	"datasource_appsec":     false,
+	"datasource_cloudwatch": false,
+	"datasource_docker":     false,
+	"datasource_file":       false,
+	"datasource_journalctl": false,
+	"datasource_k8s-audit":  false,
+	"datasource_kafka":      false,
+	"datasource_kinesis":    false,
+	"datasource_loki":       false,
+	"datasource_s3":         false,
+	"datasource_syslog":     false,
+	"datasource_wineventlog":false,
+	"cscli_setup":           false,
+}
+
+func Register(name string) {
+	if _, ok := Built[name]; !ok {
+		// having a list of the disabled components is essential
+		// to debug users' issues
+		panic("cannot register unknown compile-time component: " + name)
+	}
+
+	Built[name] = true
+}

pkg/cwversion/version.go

@@ -7,8 +7,8 @@ import (
 	"github.com/crowdsecurity/go-cs-lib/maptools"
 	"github.com/crowdsecurity/go-cs-lib/version"
 
-	"github.com/crowdsecurity/crowdsec/pkg/acquisition"
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient/useragent"
+	"github.com/crowdsecurity/crowdsec/pkg/cwversion/component"
 	"github.com/crowdsecurity/crowdsec/pkg/cwversion/constraint"
 )
 
@@ -18,16 +18,16 @@ var (
 )
 
 func FullString() string {
-	dsBuilt := []string{}
-	dsExcluded := []string{}
+	dsBuilt := map[string]struct{}{}
+	dsExcluded := map[string]struct{}{}
 
-	for _, ds := range maptools.SortedKeys(acquisition.AcquisitionSources) {
-		if acquisition.AcquisitionSources[ds] != nil {
-			dsBuilt = append(dsBuilt, ds)
+	for ds, built := range component.Built {
+		if built {
+			dsBuilt[ds] = struct{}{}
 			continue
 		}
 
-		dsExcluded = append(dsExcluded, ds)
+		dsExcluded[ds] = struct{}{}
 	}
 
 	ret := fmt.Sprintf("version: %s\n", version.String())
@@ -42,12 +42,16 @@ func FullString() string {
 	ret += fmt.Sprintf("Constraint_api: %s\n", constraint.API)
 	ret += fmt.Sprintf("Constraint_acquis: %s\n", constraint.Acquis)
 
+	built := "(none)"
+
 	if len(dsBuilt) > 0 {
-		ret += fmt.Sprintf("Built data sources: %s\n", strings.Join(dsBuilt, ", "))
+		built = strings.Join(maptools.SortedKeys(dsBuilt), ", ")
 	}
 
+	ret += fmt.Sprintf("Built-in optional components: %s\n", built)
+
 	if len(dsExcluded) > 0 {
-		ret += fmt.Sprintf("Excluded data sources: %s\n", strings.Join(dsExcluded, ", "))
+		ret += fmt.Sprintf("Excluded components: %s\n", strings.Join(maptools.SortedKeys(dsExcluded), ", "))
 	}
 
 	return ret