Add explicit configuration for signals sharing and blocklists pull (#3277)

cmd/crowdsec-cli/clicapi/capi.go

@@ -225,6 +225,27 @@ func (cli *cliCapi) Status(ctx context.Context, out io.Writer, hub *cwhub.Hub) e
 		fmt.Fprint(out, "Your instance is enrolled in the console\n")
 	}
 
+	switch *cfg.API.Server.OnlineClient.Sharing {
+	case true:
+		fmt.Fprint(out, "Sharing signals is enabled\n")
+	case false:
+		fmt.Fprint(out, "Sharing signals is disabled\n")
+	}
+
+	switch *cfg.API.Server.OnlineClient.PullConfig.Community {
+	case true:
+		fmt.Fprint(out, "Pulling community blocklist is enabled\n")
+	case false:
+		fmt.Fprint(out, "Pulling community blocklist is disabled\n")
+	}
+
+	switch *cfg.API.Server.OnlineClient.PullConfig.Blocklists {
+	case true:
+		fmt.Fprint(out, "Pulling blocklists from the console is enabled\n")
+	case false:
+		fmt.Fprint(out, "Pulling blocklists from the console is disabled\n")
+	}
+
 	return nil
 }
 

pkg/apiclient/decisions_service.go

@@ -31,6 +31,8 @@ type DecisionsListOpts struct {
 
 type DecisionsStreamOpts struct {
 	Startup                bool   `url:"startup,omitempty"`
+	CommunityPull          bool   `url:"community_pull"`
+	AdditionalPull         bool   `url:"additional_pull"`
 	Scopes                 string `url:"scopes,omitempty"`
 	ScenariosContaining    string `url:"scenarios_containing,omitempty"`
 	ScenariosNotContaining string `url:"scenarios_not_containing,omitempty"`
@@ -43,6 +45,17 @@ func (o *DecisionsStreamOpts) addQueryParamsToURL(url string) (string, error) {
 		return "", err
 	}
 
+	//Those 2 are a bit different
+	//They default to true, and we only want to include them if they are false
+
+	if params.Get("community_pull") == "true" {
+		params.Del("community_pull")
+	}
+
+	if params.Get("additional_pull") == "true" {
+		params.Del("additional_pull")
+	}
+
 	return fmt.Sprintf("%s?%s", url, params.Encode()), nil
 }
 

pkg/apiclient/decisions_service_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"net/http"
 	"net/url"
+	"strings"
 	"testing"
 
 	log "github.com/sirupsen/logrus"
@@ -87,7 +88,7 @@ func TestDecisionsStream(t *testing.T) {
 		testMethod(t, r, http.MethodGet)
 
 		if r.Method == http.MethodGet {
-			if r.URL.RawQuery == "startup=true" {
+			if strings.Contains(r.URL.RawQuery, "startup=true") {
 				w.WriteHeader(http.StatusOK)
 				w.Write([]byte(`{"deleted":null,"new":[{"duration":"3h59m55.756182786s","id":4,"origin":"cscli","scenario":"manual 'ban' from '82929df7ee394b73b81252fe3b4e50203yaT2u6nXiaN7Ix9'","scope":"Ip","type":"ban","value":"1.2.3.4"}]}`))
 			} else {
@@ -160,7 +161,7 @@ func TestDecisionsStreamV3Compatibility(t *testing.T) {
 		testMethod(t, r, http.MethodGet)
 
 		if r.Method == http.MethodGet {
-			if r.URL.RawQuery == "startup=true" {
+			if strings.Contains(r.URL.RawQuery, "startup=true") {
 				w.WriteHeader(http.StatusOK)
 				w.Write([]byte(`{"deleted":[{"scope":"ip","decisions":["1.2.3.5"]}],"new":[{"scope":"ip", "scenario": "manual 'ban' from '82929df7ee394b73b81252fe3b4e50203yaT2u6nXiaN7Ix9'", "decisions":[{"duration":"3h59m55.756182786s","value":"1.2.3.4"}]}]}`))
 			} else {
@@ -429,6 +430,8 @@ func TestDecisionsStreamOpts_addQueryParamsToURL(t *testing.T) {
 		Scopes                 string
 		ScenariosContaining    string
 		ScenariosNotContaining string
+		CommunityPull          bool
+		AdditionalPull         bool
 	}
 
 	tests := []struct {
@@ -440,11 +443,17 @@ func TestDecisionsStreamOpts_addQueryParamsToURL(t *testing.T) {
 		{
 			name:     "no filter",
 			expected: baseURLString + "?",
+			fields: fields{
+				CommunityPull:  true,
+				AdditionalPull: true,
+			},
 		},
 		{
 			name: "startup=true",
 			fields: fields{
-				Startup: true,
+				Startup:        true,
+				CommunityPull:  true,
+				AdditionalPull: true,
 			},
 			expected: baseURLString + "?startup=true",
 		},
@@ -455,9 +464,19 @@ func TestDecisionsStreamOpts_addQueryParamsToURL(t *testing.T) {
 				Scopes:                 "ip,range",
 				ScenariosContaining:    "ssh",
 				ScenariosNotContaining: "bf",
+				CommunityPull:          true,
+				AdditionalPull:         true,
 			},
 			expected: baseURLString + "?scenarios_containing=ssh&scenarios_not_containing=bf&scopes=ip%2Crange&startup=true",
 		},
+		{
+			name: "pull options",
+			fields: fields{
+				CommunityPull:  false,
+				AdditionalPull: false,
+			},
+			expected: baseURLString + "?additional_pull=false&community_pull=false",
+		},
 	}
 
 	for _, tt := range tests {
@@ -467,6 +486,8 @@ func TestDecisionsStreamOpts_addQueryParamsToURL(t *testing.T) {
 				Scopes:                 tt.fields.Scopes,
 				ScenariosContaining:    tt.fields.ScenariosContaining,
 				ScenariosNotContaining: tt.fields.ScenariosNotContaining,
+				CommunityPull:          tt.fields.CommunityPull,
+				AdditionalPull:         tt.fields.AdditionalPull,
 			}
 
 			got, err := o.addQueryParamsToURL(baseURLString)

pkg/apiserver/apic.go

@@ -69,6 +69,10 @@ type apic struct {
 	consoleConfig *csconfig.ConsoleConfig
 	isPulling     chan bool
 	whitelists    *csconfig.CapiWhitelist
+
+	pullBlocklists bool
+	pullCommunity  bool
+	shareSignals   bool
 }
 
 // randomDuration returns a duration value between d-delta and d+delta
@@ -198,6 +202,9 @@ func NewAPIC(ctx context.Context, config *csconfig.OnlineApiClientCfg, dbClient
 		usageMetricsIntervalFirst: randomDuration(usageMetricsInterval, usageMetricsIntervalDelta),
 		isPulling:                 make(chan bool, 1),
 		whitelists:                apicWhitelist,
+		pullBlocklists:            *config.PullConfig.Blocklists,
+		pullCommunity:             *config.PullConfig.Community,
+		shareSignals:              *config.Sharing,
 	}
 
 	password := strfmt.Password(config.Credentials.Password)
@@ -295,7 +302,7 @@ func (a *apic) Push(ctx context.Context) error {
 			var signals []*models.AddSignalsRequestItem
 
 			for _, alert := range alerts {
-				if ok := shouldShareAlert(alert, a.consoleConfig); ok {
+				if ok := shouldShareAlert(alert, a.consoleConfig, a.shareSignals); ok {
 					signals = append(signals, alertToSignal(alert, getScenarioTrustOfAlert(alert), *a.consoleConfig.ShareContext))
 				}
 			}
@@ -324,7 +331,13 @@ func getScenarioTrustOfAlert(alert *models.Alert) string {
 	return scenarioTrust
 }
 
-func shouldShareAlert(alert *models.Alert, consoleConfig *csconfig.ConsoleConfig) bool {
+func shouldShareAlert(alert *models.Alert, consoleConfig *csconfig.ConsoleConfig, shareSignals bool) bool {
+
+	if !shareSignals {
+		log.Debugf("sharing signals is disabled")
+		return false
+	}
+
 	if *alert.Simulated {
 		log.Debugf("simulation enabled for alert (id:%d), will not be sent to CAPI", alert.ID)
 		return false
@@ -625,7 +638,9 @@ func (a *apic) PullTop(ctx context.Context, forcePull bool) error {
 
 	log.Infof("Starting community-blocklist update")
 
-	data, _, err := a.apiClient.Decisions.GetStreamV3(ctx, apiclient.DecisionsStreamOpts{Startup: a.startup})
+	log.Debugf("Community pull: %t | Blocklist pull: %t", a.pullCommunity, a.pullBlocklists)
+
+	data, _, err := a.apiClient.Decisions.GetStreamV3(ctx, apiclient.DecisionsStreamOpts{Startup: a.startup, CommunityPull: a.pullCommunity, AdditionalPull: a.pullBlocklists})
 	if err != nil {
 		return fmt.Errorf("get stream: %w", err)
 	}
@@ -650,23 +665,26 @@ func (a *apic) PullTop(ctx context.Context, forcePull bool) error {
 
 	log.Printf("capi/community-blocklist : %d explicit deletions", nbDeleted)
 
-	if len(data.New) == 0 {
+	if len(data.New) > 0 {
-		log.Infof("capi/community-blocklist : received 0 new entries (expected if you just installed crowdsec)")
+		// create one alert for community blocklist using the first decision
-		return nil
+		decisions := a.apiClient.Decisions.GetDecisionsFromGroups(data.New)
-	}
+		// apply APIC specific whitelists
+		decisions = a.ApplyApicWhitelists(decisions)
 
-	// create one alert for community blocklist using the first decision
+		alert := createAlertForDecision(decisions[0])
-	decisions := a.apiClient.Decisions.GetDecisionsFromGroups(data.New)
+		alertsFromCapi := []*models.Alert{alert}
-	// apply APIC specific whitelists
+		alertsFromCapi = fillAlertsWithDecisions(alertsFromCapi, decisions, addCounters)
-	decisions = a.ApplyApicWhitelists(decisions)
 
-	alert := createAlertForDecision(decisions[0])
+		err = a.SaveAlerts(ctx, alertsFromCapi, addCounters, deleteCounters)
-	alertsFromCapi := []*models.Alert{alert}
+		if err != nil {
-	alertsFromCapi = fillAlertsWithDecisions(alertsFromCapi, decisions, addCounters)
+			return fmt.Errorf("while saving alerts: %w", err)
-
+		}
-	err = a.SaveAlerts(ctx, alertsFromCapi, addCounters, deleteCounters)
+	} else {
-	if err != nil {
+		if a.pullCommunity {
-		return fmt.Errorf("while saving alerts: %w", err)
+			log.Info("capi/community-blocklist : received 0 new entries (expected if you just installed crowdsec)")
+		} else {
+			log.Debug("capi/community-blocklist : community blocklist pull is disabled")
+		}
 	}
 
 	// update blocklists

pkg/apiserver/apic_test.go

@@ -69,7 +69,10 @@ func getAPIC(t *testing.T, ctx context.Context) *apic {
 			ShareCustomScenarios:  ptr.Of(false),
 			ShareContext:          ptr.Of(false),
 		},
-		isPulling: make(chan bool, 1),
+		isPulling:      make(chan bool, 1),
+		shareSignals:   true,
+		pullBlocklists: true,
+		pullCommunity:  true,
 	}
 }
 
@@ -200,6 +203,11 @@ func TestNewAPIC(t *testing.T) {
 				Login:    "foo",
 				Password: "bar",
 			},
+			Sharing: ptr.Of(true),
+			PullConfig: csconfig.CapiPullConfig{
+				Community:  ptr.Of(true),
+				Blocklists: ptr.Of(true),
+			},
 		}
 	}
 
@@ -1193,6 +1201,7 @@ func TestShouldShareAlert(t *testing.T) {
 	tests := []struct {
 		name          string
 		consoleConfig *csconfig.ConsoleConfig
+		shareSignals  bool
 		alert         *models.Alert
 		expectedRet   bool
 		expectedTrust string
@@ -1203,6 +1212,7 @@ func TestShouldShareAlert(t *testing.T) {
 				ShareCustomScenarios: ptr.Of(true),
 			},
 			alert:         &models.Alert{Simulated: ptr.Of(false)},
+			shareSignals:  true,
 			expectedRet:   true,
 			expectedTrust: "custom",
 		},
@@ -1212,6 +1222,7 @@ func TestShouldShareAlert(t *testing.T) {
 				ShareCustomScenarios: ptr.Of(false),
 			},
 			alert:         &models.Alert{Simulated: ptr.Of(false)},
+			shareSignals:  true,
 			expectedRet:   false,
 			expectedTrust: "custom",
 		},
@@ -1220,6 +1231,7 @@ func TestShouldShareAlert(t *testing.T) {
 			consoleConfig: &csconfig.ConsoleConfig{
 				ShareManualDecisions: ptr.Of(true),
 			},
+			shareSignals: true,
 			alert: &models.Alert{
 				Simulated: ptr.Of(false),
 				Decisions: []*models.Decision{{Origin: ptr.Of(types.CscliOrigin)}},
@@ -1232,6 +1244,7 @@ func TestShouldShareAlert(t *testing.T) {
 			consoleConfig: &csconfig.ConsoleConfig{
 				ShareManualDecisions: ptr.Of(false),
 			},
+			shareSignals: true,
 			alert: &models.Alert{
 				Simulated: ptr.Of(false),
 				Decisions: []*models.Decision{{Origin: ptr.Of(types.CscliOrigin)}},
@@ -1244,6 +1257,7 @@ func TestShouldShareAlert(t *testing.T) {
 			consoleConfig: &csconfig.ConsoleConfig{
 				ShareTaintedScenarios: ptr.Of(true),
 			},
+			shareSignals: true,
 			alert: &models.Alert{
 				Simulated:    ptr.Of(false),
 				ScenarioHash: ptr.Of("whateverHash"),
@@ -1256,6 +1270,7 @@ func TestShouldShareAlert(t *testing.T) {
 			consoleConfig: &csconfig.ConsoleConfig{
 				ShareTaintedScenarios: ptr.Of(false),
 			},
+			shareSignals: true,
 			alert: &models.Alert{
 				Simulated:    ptr.Of(false),
 				ScenarioHash: ptr.Of("whateverHash"),
@@ -1263,11 +1278,24 @@ func TestShouldShareAlert(t *testing.T) {
 			expectedRet:   false,
 			expectedTrust: "tainted",
 		},
+		{
+			name: "manual alert should not be shared if global sharing is disabled",
+			consoleConfig: &csconfig.ConsoleConfig{
+				ShareManualDecisions: ptr.Of(true),
+			},
+			shareSignals: false,
+			alert: &models.Alert{
+				Simulated:    ptr.Of(false),
+				ScenarioHash: ptr.Of("whateverHash"),
+			},
+			expectedRet:   false,
+			expectedTrust: "manual",
+		},
 	}
 
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
-			ret := shouldShareAlert(tc.alert, tc.consoleConfig)
+			ret := shouldShareAlert(tc.alert, tc.consoleConfig, tc.shareSignals)
 			assert.Equal(t, tc.expectedRet, ret)
 		})
 	}

pkg/csconfig/api.go

@@ -38,10 +38,17 @@ type ApiCredentialsCfg struct {
 	CertPath   string `yaml:"cert_path,omitempty"`
 }
 
-/*global api config (for lapi->oapi)*/
+type CapiPullConfig struct {
+	Community  *bool `yaml:"community,omitempty"`
+	Blocklists *bool `yaml:"blocklists,omitempty"`
+}
+
+/*global api config (for lapi->capi)*/
 type OnlineApiClientCfg struct {
 	CredentialsFilePath string             `yaml:"credentials_path,omitempty"` // credz will be edited by software, store in diff file
 	Credentials         *ApiCredentialsCfg `yaml:"-"`
+	PullConfig          CapiPullConfig     `yaml:"pull,omitempty"`
+	Sharing             *bool              `yaml:"sharing,omitempty"`
 }
 
 /*local api config (for crowdsec/cscli->lapi)*/
@@ -344,6 +351,21 @@ func (c *Config) LoadAPIServer(inCli bool) error {
 		log.Printf("push and pull to Central API disabled")
 	}
 
+	//Set default values for CAPI push/pull
+	if c.API.Server.OnlineClient != nil {
+		if c.API.Server.OnlineClient.PullConfig.Community == nil {
+			c.API.Server.OnlineClient.PullConfig.Community = ptr.Of(true)
+		}
+
+		if c.API.Server.OnlineClient.PullConfig.Blocklists == nil {
+			c.API.Server.OnlineClient.PullConfig.Blocklists = ptr.Of(true)
+		}
+
+		if c.API.Server.OnlineClient.Sharing == nil {
+			c.API.Server.OnlineClient.Sharing = ptr.Of(true)
+		}
+	}
+
 	if err := c.LoadDBConfig(inCli); err != nil {
 		return err
 	}

pkg/csconfig/api_test.go

@@ -212,6 +212,11 @@ func TestLoadAPIServer(t *testing.T) {
 						Login:    "test",
 						Password: "testpassword",
 					},
+					Sharing: ptr.Of(true),
+					PullConfig: CapiPullConfig{
+						Community:  ptr.Of(true),
+						Blocklists: ptr.Of(true),
+					},
 				},
 				Profiles:               tmpLAPI.Profiles,
 				ProfilesPath:           "./testdata/profiles.yaml",

pkg/modelscapi/centralapi_swagger.yaml

@@ -55,6 +55,19 @@ paths:
       description: "returns list of top decisions to add or delete"
       produces:
         - "application/json"
+      parameters:
+        - in: query
+          name: "community_pull"
+          type: "boolean"
+          default: true
+          required: false
+          description: "Fetch the community blocklist content"
+        - in: query
+          name: "additional_pull"
+          type: "boolean"
+          default: true
+          required: false
+          description: "Fetch additional blocklists content"
       responses:
         "200":
           description: "200 response"