From c4ff4228befb022df10c16a22b7406f9e307c095 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Wed, 19 Feb 2025 14:05:17 +0100 Subject: [PATCH] use go 1.24, enable unencrypted http2 (#3470) --- .github/workflows/go-tests-windows.yml | 2 +- .github/workflows/go-tests.yml | 2 +- .golangci.yml | 7 ++++++- Dockerfile | 2 +- Dockerfile.debian | 2 +- go.mod | 6 +----- pkg/acquisition/modules/appsec/appsec.go | 9 +++++++-- pkg/acquisition/modules/http/http.go | 9 +++++++-- .../modules/kubernetesaudit/k8s_audit.go | 10 ++++++++-- pkg/apiserver/apiserver.go | 8 ++++++-- pkg/exprhelpers/debugger.go | 4 ++-- pkg/exprhelpers/helpers.go | 18 ++++++++++++++++-- test/bats/08_metrics_bouncer.bats | 2 +- 13 files changed, 58 insertions(+), 23 deletions(-) diff --git a/.github/workflows/go-tests-windows.yml b/.github/workflows/go-tests-windows.yml index 68cb9715b..a99572fee 100644 --- a/.github/workflows/go-tests-windows.yml +++ b/.github/workflows/go-tests-windows.yml @@ -61,6 +61,6 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v6 with: - version: v1.63 + version: v1.64 args: --issues-exit-code=1 --timeout 10m only-new-issues: false diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 8629f58cf..6a48a932b 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -198,6 +198,6 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v6 with: - version: v1.63 + version: v1.64 args: --issues-exit-code=1 --timeout 10m only-new-issues: false diff --git a/.golangci.yml b/.golangci.yml index 3afa4571b..ede7de421 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -240,7 +240,7 @@ linters: # # DEPRECATED by golangi-lint # - - exportloopref + - tenv # # Redundant @@ -493,6 +493,11 @@ issues: text: "argument-limit: .*" # need some cleanup first: to create db in memory and share the client, not the config + - linters: + - usetesting + path: "(.+)_test.go" + text: "context.Background.*" + - linters: - usetesting path: "pkg/apiserver/(.+)_test.go" diff --git a/Dockerfile b/Dockerfile index 383578c48..45b9c2bed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/golang:1.23-alpine3.21 AS build +FROM docker.io/golang:1.24-alpine3.21 AS build ARG BUILD_VERSION diff --git a/Dockerfile.debian b/Dockerfile.debian index a9b58c633..0e99ade78 100644 --- a/Dockerfile.debian +++ b/Dockerfile.debian @@ -1,4 +1,4 @@ -FROM docker.io/golang:1.23-bookworm AS build +FROM docker.io/golang:1.24-bookworm AS build ARG BUILD_VERSION diff --git a/go.mod b/go.mod index 63b0a1893..8c12c0292 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,6 @@ module github.com/crowdsecurity/crowdsec -go 1.23.6 - -// Don't use the toolchain directive to avoid uncontrolled downloads during -// a build, especially in sandboxed environments (freebsd, gentoo...). -// toolchain go1.21.3 +go 1.24.0 require ( entgo.io/ent v0.14.2 diff --git a/pkg/acquisition/modules/appsec/appsec.go b/pkg/acquisition/modules/appsec/appsec.go index 78225d5f8..3e2a4f576 100644 --- a/pkg/acquisition/modules/appsec/appsec.go +++ b/pkg/acquisition/modules/appsec/appsec.go @@ -173,10 +173,15 @@ func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLe w.mux = http.NewServeMux() w.server = &http.Server{ - Addr: w.config.ListenAddr, - Handler: w.mux, + Addr: w.config.ListenAddr, + Handler: w.mux, + Protocols: &http.Protocols{}, } + w.server.Protocols.SetHTTP1(true) + w.server.Protocols.SetUnencryptedHTTP2(true) + w.server.Protocols.SetHTTP2(true) + w.InChan = make(chan appsec.ParsedRequest) appsecCfg := appsec.AppsecConfig{Logger: w.logger.WithField("component", "appsec_config")} diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index 97e220570..76d7d06d2 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -372,10 +372,15 @@ func (h *HTTPSource) RunServer(out chan types.Event, t *tomb.Tomb) error { }) h.Server = &http.Server{ - Addr: h.Config.ListenAddr, - Handler: mux, + Addr: h.Config.ListenAddr, + Handler: mux, + Protocols: &http.Protocols{}, } + h.Server.Protocols.SetHTTP1(true) + h.Server.Protocols.SetUnencryptedHTTP2(true) + h.Server.Protocols.SetHTTP2(true) + if h.Config.Timeout != nil { h.Server.ReadTimeout = *h.Config.Timeout } diff --git a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go index b0650d390..5d1d04c95 100644 --- a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go +++ b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go @@ -113,10 +113,15 @@ func (ka *KubernetesAuditSource) Configure(config []byte, logger *log.Entry, met ka.mux = http.NewServeMux() ka.server = &http.Server{ - Addr: ka.addr, - Handler: ka.mux, + Addr: ka.addr, + Handler: ka.mux, + Protocols: &http.Protocols{}, } + ka.server.Protocols.SetHTTP1(true) + ka.server.Protocols.SetUnencryptedHTTP2(true) + ka.server.Protocols.SetHTTP2(true) + ka.mux.HandleFunc(ka.config.WebhookPath, ka.webhookHandler) return nil @@ -154,6 +159,7 @@ func (ka *KubernetesAuditSource) StreamingAcquisition(ctx context.Context, out c }) <-t.Dying() ka.logger.Infof("Stopping k8s-audit server on %s:%d%s", ka.config.ListenAddr, ka.config.ListenPort, ka.config.WebhookPath) + if err := ka.server.Shutdown(ctx); err != nil { ka.logger.Errorf("Error shutting down k8s-audit server: %s", err.Error()) } diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 88f1bd21d..a9ab45ceb 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -56,8 +56,7 @@ func isBrokenConnection(maybeError any) bool { if errors.As(err, &netOpError) { var syscallError *os.SyscallError if errors.As(netOpError.Err, &syscallError) { - if strings.Contains(strings.ToLower(syscallError.Error()), "broken pipe") || - strings.Contains(strings.ToLower(syscallError.Error()), "connection reset by peer") { + if strings.Contains(strings.ToLower(syscallError.Error()), "broken pipe") || strings.Contains(strings.ToLower(syscallError.Error()), "connection reset by peer") { return true } } @@ -384,8 +383,13 @@ func (s *APIServer) Run(apiReady chan bool) error { Addr: s.URL, Handler: s.router, TLSConfig: tlsCfg, + Protocols: &http.Protocols{}, } + s.httpServer.Protocols.SetHTTP1(true) + s.httpServer.Protocols.SetUnencryptedHTTP2(true) + s.httpServer.Protocols.SetHTTP2(true) + ctx := context.TODO() if s.apic != nil { diff --git a/pkg/exprhelpers/debugger.go b/pkg/exprhelpers/debugger.go index d2c76e05a..65aa29b6a 100644 --- a/pkg/exprhelpers/debugger.go +++ b/pkg/exprhelpers/debugger.go @@ -149,9 +149,9 @@ func autoQuote(v any) string { // let's avoid printing long strings. it can happen ie. when we are debugging expr with `File()` or similar helpers if len(x) > 40 { return fmt.Sprintf("%q", x[:40]+"...") - } else { - return fmt.Sprintf("%q", x) } + + return fmt.Sprintf("%q", x) default: return fmt.Sprintf("%v", x) } diff --git a/pkg/exprhelpers/helpers.go b/pkg/exprhelpers/helpers.go index d0f6f2cfe..3525bb6c7 100644 --- a/pkg/exprhelpers/helpers.go +++ b/pkg/exprhelpers/helpers.go @@ -216,7 +216,8 @@ func FileInit(fileFolder string, filename string, fileType string) error { if strings.HasPrefix(scanner.Text(), "#") { // allow comments continue } - if scanner.Text() == "" { //skip empty lines + + if scanner.Text() == "" { // skip empty lines continue } @@ -262,7 +263,7 @@ func Distinct(params ...any) (any, error) { } func FlattenDistinct(params ...any) (any, error) { - return Distinct(flatten(nil, reflect.ValueOf(params))) //nolint:asasalint + return Distinct(flatten(nil, reflect.ValueOf(params))) } func Flatten(params ...any) (any, error) { @@ -312,9 +313,11 @@ func existsInFileMaps(filename string, ftype string) (bool, error) { func Get(params ...any) (any, error) { arr := params[0].([]string) index := params[1].(int) + if index >= len(arr) { return "", nil } + return arr[index], nil } @@ -407,22 +410,26 @@ func PathEscape(params ...any) (any, error) { // func PathUnescape(s string) string { func PathUnescape(params ...any) (any, error) { s := params[0].(string) + ret, err := url.PathUnescape(s) if err != nil { log.Debugf("unable to PathUnescape '%s': %+v", s, err) return s, nil } + return ret, nil } // func QueryUnescape(s string) string { func QueryUnescape(params ...any) (any, error) { s := params[0].(string) + ret, err := url.QueryUnescape(s) if err != nil { log.Debugf("unable to QueryUnescape '%s': %+v", s, err) return s, nil } + return ret, nil } @@ -432,8 +439,10 @@ func File(params ...any) (any, error) { if _, ok := dataFile[filename]; ok { return dataFile[filename], nil } + log.Errorf("file '%s' (type:string) not found in expr library", filename) log.Errorf("expr library : %s", spew.Sdump(dataFile)) + return []string{}, nil } @@ -441,13 +450,16 @@ func File(params ...any) (any, error) { func RegexpInFile(params ...any) (any, error) { data := params[0].(string) filename := params[1].(string) + var hash uint64 + hasCache := false matched := false if _, ok := dataFileRegexCache[filename]; ok { hasCache = true hash = xxhash.Sum64String(data) + if val, err := dataFileRegexCache[filename].Get(hash); err == nil { return val.(bool), nil } @@ -479,9 +491,11 @@ func RegexpInFile(params ...any) (any, error) { log.Errorf("expr library : %s", spew.Sdump(dataFileRegex)) } } + if hasCache { dataFileRegexCache[filename].Set(hash, matched) } + return matched, nil } diff --git a/test/bats/08_metrics_bouncer.bats b/test/bats/08_metrics_bouncer.bats index 5fb2c543b..1c9e93957 100644 --- a/test/bats/08_metrics_bouncer.bats +++ b/test/bats/08_metrics_bouncer.bats @@ -74,7 +74,7 @@ teardown() { payload=$(yq -o j '.remediation_components[0].utc_startup_timestamp = "2021-09-01T00:00:00Z"' <<<"$payload") rune -22 curl-with-key '/v1/usage-metrics' -X POST --data "$payload" assert_stderr --partial "error: 400" - assert_json '{message: "json: cannot unmarshal string into Go struct field AllMetrics.remediation_components of type int64"}' + assert_json '{message: "json: cannot unmarshal string into Go struct field AllMetrics.remediation_components.utc_startup_timestamp of type int64"}' payload=$(yq -o j '.remediation_components[0].utc_startup_timestamp = 1707399316' <<<"$payload") rune -0 curl-with-key '/v1/usage-metrics' -X POST --data "$payload"