crowdsec: allow -t to work if using appsec and allowlists (#3484)

pkg/acquisition/modules/appsec/appsec.go

@@ -63,7 +63,6 @@ type AppsecSource struct {
 	lapiURL               string
 	AuthCache             AuthCache
 	AppsecRunners         []AppsecRunner // one for each go-routine
-	apiClient             *apiclient.ApiClient
 	appsecAllowlistClient *allowlists.AppsecAllowlist
 }
 
@@ -226,12 +225,7 @@ func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLe
 
 	w.AppsecRunners = make([]AppsecRunner, w.config.Routines)
 
-	w.apiClient, err = apiclient.GetLAPIClient()
-	if err != nil {
-		return fmt.Errorf("unable to get authenticated LAPI client: %w", err)
-	}
-
-	w.appsecAllowlistClient = allowlists.NewAppsecAllowlist(w.apiClient, w.logger)
+	w.appsecAllowlistClient = allowlists.NewAppsecAllowlist(w.logger)
 
 	for nbRoutine := range w.config.Routines {
 		appsecRunnerUUID := uuid.New().String()
@@ -282,7 +276,16 @@ func (w *AppsecSource) OneShotAcquisition(_ context.Context, _ chan types.Event,
 func (w *AppsecSource) StreamingAcquisition(ctx context.Context, out chan types.Event, t *tomb.Tomb) error {
 	w.outChan = out
 
-	w.appsecAllowlistClient.StartRefresh(t)
+	apiClient, err := apiclient.GetLAPIClient()
+	if err != nil {
+		return fmt.Errorf("unable to get authenticated LAPI client: %w", err)
+	}
+
+	err = w.appsecAllowlistClient.Start(ctx, apiClient)
+	if err != nil {
+		return fmt.Errorf("failed to fetch allowlists: %w", err)
+	}
+	w.appsecAllowlistClient.StartRefresh(ctx, t)
 
 	t.Go(func() error {
 		defer trace.CatchPanic("crowdsec/acquis/appsec/live")

pkg/acquisition/modules/appsec/appsec_test.go

@@ -146,10 +146,9 @@ func loadAppSecEngine(test appsecRuleTest, t *testing.T) {
 		assert.NoError(t, err)
 	})
 
-	allowlistClient := allowlists.NewAppsecAllowlist(client, logger)
-	// In real life, allowlists updater is started by the acquisition
-	// Do it manually here as we are simulating the appsec itself
-	err = allowlistClient.FetchAllowlists()
+	allowlistClient := allowlists.NewAppsecAllowlist(logger)
+
+	err = allowlistClient.Start(t.Context(), client)
 	require.NoError(t, err)
 	runner := AppsecRunner{
 		inChan:                 InChan,

pkg/appsec/allowlists/allowlists.go

@@ -36,25 +36,26 @@ type AppsecAllowlist struct {
 	tomb       *tomb.Tomb
 }
 
-func NewAppsecAllowlist(client *apiclient.ApiClient, logger *log.Entry) *AppsecAllowlist {
+func NewAppsecAllowlist(logger *log.Entry) *AppsecAllowlist {
 	a := &AppsecAllowlist{
-		LAPIClient: client,
-		logger:     logger.WithField("component", "appsec-allowlist"),
-		ips:        []ipAllowlist{},
-		ranges:     []rangeAllowlist{},
-	}
-
-	if err := a.FetchAllowlists(); err != nil {
-		a.logger.Errorf("failed to fetch allowlists: %s", err)
+		logger: logger.WithField("component", "appsec-allowlist"),
+		ips:    []ipAllowlist{},
+		ranges: []rangeAllowlist{},
 	}
 
 	return a
 }
 
-func (a *AppsecAllowlist) FetchAllowlists() error {
+func (a *AppsecAllowlist) Start(ctx context.Context, client *apiclient.ApiClient) error {
+	a.LAPIClient = client
+	err := a.FetchAllowlists(ctx)
+	return err
+}
+
+func (a *AppsecAllowlist) FetchAllowlists(ctx context.Context) error {
 	a.logger.Debug("fetching allowlists")
 
-	allowlists, _, err := a.LAPIClient.Allowlists.List(context.TODO(), apiclient.AllowlistListOpts{WithContent: true})
+	allowlists, _, err := a.LAPIClient.Allowlists.List(ctx, apiclient.AllowlistListOpts{WithContent: true})
 	if err != nil {
 		return err
 	}
@@ -92,6 +93,9 @@ func (a *AppsecAllowlist) FetchAllowlists() error {
 		}
 	}
 
+	if len(a.ips) != 0 || len(a.ranges) != 0 {
+		a.logger.Infof("fetched %d IPs and %d ranges", len(a.ips), len(a.ranges))
+	}
 	a.logger.Debugf("fetched %d IPs and %d ranges", len(a.ips), len(a.ranges))
 	a.logger.Tracef("allowlisted ips: %+v", a.ips)
 	a.logger.Tracef("allowlisted ranges: %+v", a.ranges)
@@ -99,25 +103,28 @@ func (a *AppsecAllowlist) FetchAllowlists() error {
 	return nil
 }
 
-func (a *AppsecAllowlist) updateAllowlists() error {
+func (a *AppsecAllowlist) updateAllowlists(ctx context.Context) {
 	ticker := time.NewTicker(allowlistRefreshInterval)
 
 	for {
 		select {
 		case <-ticker.C:
-			if err := a.FetchAllowlists(); err != nil {
+			if err := a.FetchAllowlists(ctx); err != nil {
 				a.logger.Errorf("failed to fetch allowlists: %s", err)
 			}
 		case <-a.tomb.Dying():
 			ticker.Stop()
-			return nil
+			return
 		}
 	}
 }
 
-func (a *AppsecAllowlist) StartRefresh(t *tomb.Tomb) {
+func (a *AppsecAllowlist) StartRefresh(ctx context.Context, t *tomb.Tomb) {
 	a.tomb = t
-	a.tomb.Go(a.updateAllowlists)
+	a.tomb.Go(func() error {
+		a.updateAllowlists(ctx)
+		return nil
+	})
 }
 
 func (a *AppsecAllowlist) IsAllowlisted(sourceIP string) (bool, string) {

pkg/appsec/allowlists/allowlists_test.go

@@ -64,9 +64,13 @@ func TestAppsecAllowlist(t *testing.T) {
 		assert.NoError(t, err)
 	})
 
-	allowlistClient := NewAppsecAllowlist(client, log.NewEntry(log.StandardLogger()))
+	ctx := t.Context()
+	allowlistClient := NewAppsecAllowlist(log.NewEntry(log.StandardLogger()))
 
-	err = allowlistClient.FetchAllowlists()
+	err = allowlistClient.Start(ctx, client)
+	require.NoError(t, err)
+
+	err = allowlistClient.FetchAllowlists(ctx)
 	require.NoError(t, err)
 
 	res, reason := allowlistClient.IsAllowlisted("1.2.3.4")
@@ -84,7 +88,7 @@ func TestAppsecAllowlist(t *testing.T) {
 	assert.Len(t, allowlistClient.ips, 1)
 	assert.Len(t, allowlistClient.ranges, 1)
 
-	err = allowlistClient.FetchAllowlists()
+	err = allowlistClient.FetchAllowlists(ctx)
 	require.NoError(t, err)
 
 	// No duplicates should be added

test/bats/crowdsec-acquisition.bats

@@ -76,3 +76,19 @@ teardown() {
     assert_stderr --partial "datasource type missing in $ACQUIS_DIR/journal.yaml (position 0): detected 'source=journalctl'"
     assert_stderr --partial "datasource type mismatch in $ACQUIS_DIR/bad.yaml (position 0): found 'docker' but should probably be 'journalctl'"
 }
+
+@test "test mode does not fail because of appsec and allowlists" {
+    rune -0 cscli collections install crowdsecurity/appsec-virtual-patching
+    cat >"$ACQUIS_DIR/appsec.yaml" <<-EOT
+	source: appsec
+	appsec_config: crowdsecurity/virtual-patching
+	labels:
+	  type: appsec
+	EOT
+
+    config_set '.common.log_level="debug" | .common.log_media="stdout"'
+
+    rune -0 "$CROWDSEC" -t --trace
+
+    assert_stderr --partial "Configuration test done"
+}