add JA4H expr helper (#3401)

pkg/alertcontext/alertcontext_test.go

@@ -347,6 +347,42 @@ func TestAppsecEventToContext(t *testing.T) {
 			},
 			expectedErrLen: 0,
 		},
+		{
+			name: "test JA4H - appsec event",
+			contextToSend: map[string][]string{
+				"ja4h": {"JA4H(req)"},
+			},
+			match: types.AppsecEvent{
+				MatchedRules: types.MatchedRules{
+					{
+						"id": "test",
+					},
+				},
+			},
+			req: &http.Request{
+				Header: map[string][]string{
+					"User-Agent": {"test"},
+					"Foobar":     {"test1", "test2"},
+				},
+				ProtoMajor: 1,
+				ProtoMinor: 1,
+				Method:     http.MethodGet,
+			},
+			expectedResult: []*models.MetaItems0{
+				{
+					Key:   "ja4h",
+					Value: "[\"ge11nn020000_3a31a0f8fbf9_000000000000_000000000000\"]",
+				},
+			},
+		},
+		{
+			name: "test JA4H - no appsec event",
+			contextToSend: map[string][]string{
+				"ja4h": {"JA4H(req)"},
+			},
+			req:            nil,
+			expectedResult: []*models.MetaItems0{},
+		},
 	}
 
 	for _, test := range tests {

pkg/appsec/ja4h/ja4h.go

@@ -0,0 +1,185 @@
+package ja4h
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"net/http"
+	"slices"
+	"sort"
+	"strings"
+)
+
+// see: https://github.com/FoxIO-LLC/ja4/blob/main/technical_details/JA4H.png
+// [JA4H_a]_[JA4H_b]_[JA4H_c]_[JA4H_d]
+
+// JA4H_a:
+// [httpMethod] [httpVersion] [hasCookie] [hasReferer] [countHeaders] [primaryLanguage]
+//  2         2         1         1            2              4               12
+
+// JA4H_b: [headers hash]
+
+// JA4H_c: [cookie name hash]
+
+const (
+	truncatedHashLength = 12
+	ja4hFullHashLength  = 51
+	ja4hSubHashLength   = 12
+	defaultLang         = "0000"
+	emptyCookiesHash    = "000000000000"
+)
+
+// httpMethod extracts the first two lowercase characters of the HTTP method.
+func httpMethod(method string) string {
+	l := min(len(method), 2)
+	return strings.ToLower(method[:l])
+}
+
+// httpVersion extracts the version number from the HTTP protocol.
+// The version is truncated to one digit each, but I believe the  http server will control this anyway.
+func httpVersion(major int, minor int) string {
+	return fmt.Sprintf("%d%d", major%10, minor%10)
+}
+
+// hasCookie checks if the request has any cookies.
+func hasCookie(req *http.Request) string {
+	if len(req.Cookies()) > 0 {
+		return "c"
+	}
+	return "n"
+}
+
+// hasReferer checks if the Referer header is set.
+func hasReferer(referer string) string {
+	if referer != "" {
+		return "r"
+	}
+	return "n"
+}
+
+// countHeaders counts the headers, excluding specific ones like Cookie and Referer.
+func countHeaders(headers http.Header) string {
+	count := len(headers)
+	if headers.Get("Cookie") != "" {
+		count--
+	}
+	if headers.Get("Referer") != "" {
+		count--
+	}
+	//header len needs to be on 2 chars: 3 -> 03 // 100 -> 99
+	return fmt.Sprintf("%02d", min(count, 99))
+}
+
+// primaryLanguage extracts the first four characters of the primary Accept-Language header.
+func primaryLanguage(headers http.Header) string {
+	lang := strings.ToLower(headers.Get("Accept-Language"))
+	if lang == "" {
+		return defaultLang
+	}
+	//cf. https://github.com/FoxIO-LLC/ja4/blob/main/python/ja4h.py#L13
+	lang = strings.ReplaceAll(lang, "-", "")
+	lang = strings.ReplaceAll(lang, ";", ",")
+
+	value := strings.Split(lang, ",")[0]
+	value = value[:min(len(value), 4)]
+	return value + strings.Repeat("0", 4-len(value))
+}
+
+// jA4H_a generates a summary fingerprint for the HTTP request.
+func jA4H_a(req *http.Request) string {
+	var builder strings.Builder
+
+	builder.Grow(ja4hSubHashLength)
+	builder.WriteString(httpMethod(req.Method))
+	builder.WriteString(httpVersion(req.ProtoMajor, req.ProtoMinor))
+	builder.WriteString(hasCookie(req))
+	builder.WriteString(hasReferer(req.Referer()))
+	builder.WriteString(countHeaders(req.Header))
+	builder.WriteString(primaryLanguage(req.Header))
+	return builder.String()
+}
+
+// jA4H_b computes a truncated SHA256 hash of sorted header names.
+func jA4H_b(req *http.Request) string {
+
+	// The reference implementation (https://github.com/FoxIO-LLC/ja4/blob/main/python/ja4h.py#L27)
+	// discards referer and headers **starting with "cookie"**
+	// If there's no headers, it hashes the empty string, instead of returning 0s
+	// like what is done for cookies. Not sure if it's intended or an oversight in the spec.
+	headers := make([]string, 0, len(req.Header))
+	for name := range req.Header {
+		if strings.HasPrefix(strings.ToLower(name), "cookie") || strings.ToLower(name) == "referer" {
+			continue
+		}
+		headers = append(headers, name)
+	}
+	sort.Strings(headers)
+
+	return hashTruncated(strings.Join(headers, ","))
+}
+
+// hashTruncated computes a truncated SHA256 hash for the given input.
+func hashTruncated(input string) string {
+	hash := sha256.Sum256([]byte(input))
+	return fmt.Sprintf("%x", hash)[:truncatedHashLength]
+}
+
+// jA4H_c computes a truncated SHA256 hash of sorted cookie names.
+func jA4H_c(cookies []*http.Cookie) string {
+	if len(cookies) == 0 {
+		return emptyCookiesHash
+	}
+	var builder strings.Builder
+	for i, cookie := range cookies {
+		builder.WriteString(cookie.Name)
+		if i < len(cookies)-1 {
+			builder.WriteString(",")
+		}
+	}
+	return hashTruncated(builder.String())
+}
+
+// jA4H_d computes a truncated SHA256 hash of cookie name-value pairs.
+func jA4H_d(cookies []*http.Cookie) string {
+	if len(cookies) == 0 {
+		return emptyCookiesHash
+	}
+	var builder strings.Builder
+	for i, cookie := range cookies {
+		builder.WriteString(cookie.Name)
+		builder.WriteString("=")
+		builder.WriteString(cookie.Value)
+		if i < len(cookies)-1 {
+			builder.WriteString(",")
+		}
+	}
+	return hashTruncated(builder.String())
+}
+
+// JA4H computes the complete HTTP client fingerprint based on the request.
+func JA4H(req *http.Request) string {
+	JA4H_a := jA4H_a(req)
+	JA4H_b := jA4H_b(req)
+
+	cookies := req.Cookies()
+
+	slices.SortFunc(cookies, func(a, b *http.Cookie) int {
+		return strings.Compare(a.Name, b.Name)
+	})
+
+	JA4H_c := jA4H_c(cookies)
+	JA4H_d := jA4H_d(cookies)
+
+	var builder strings.Builder
+
+	//JA4H is a fixed size, allocated it all at once
+	builder.Grow(ja4hFullHashLength)
+	builder.WriteString(JA4H_a)
+	builder.WriteString("_")
+	builder.WriteString(JA4H_b)
+	builder.WriteString("_")
+	builder.WriteString(JA4H_c)
+	builder.WriteString("_")
+	builder.WriteString(JA4H_d)
+
+	return builder.String()
+}

pkg/appsec/ja4h/ja4h_test.go

@@ -0,0 +1,306 @@
+package ja4h
+
+import (
+	"net/http"
+	"slices"
+	"strings"
+	"testing"
+)
+
+/*
+
+The various hashes used comes from the python reference implementation: https://github.com/FoxIO-LLC/ja4/tree/main/python
+
+They are generated by:
+ - running a packet capture locally: sudo tshark -i lo -f "port 80" -w /tmp/foo.pcapng
+ - make a curl request: curl -b foo=bar -b baz=qux localhost
+ - generate the hash with the reference implementation: python ja4.py /tmp/foo.pcapng -r
+
+For the JA4H_B hash, the value we use is *not* the one returned by the reference implementation, as we cannot know the order of the headers.
+For those hashes, the value used was the one returned by our code (because we deviate from the spec, as long as we are consistent, it's fine).
+*/
+
+func TestJA4H_A(t *testing.T) {
+	tests := []struct {
+		name           string
+		request        func() *http.Request
+		expectedResult string
+	}{
+		{
+			name: "basic GET request - HTTP1.1 - no accept-language header",
+			request: func() *http.Request {
+				req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil)
+				return req
+			},
+			expectedResult: "ge11nn000000",
+		},
+		{
+			name: "basic GET request - HTTP1.1 - with accept-language header",
+			request: func() *http.Request {
+				req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil)
+				req.Header.Set("Accept-Language", "en-US")
+				return req
+			},
+			expectedResult: "ge11nn01enus",
+		},
+		{
+			name: "basic POST request - HTTP1.1 - no accept-language header - cookies - referer",
+			request: func() *http.Request {
+				req, _ := http.NewRequest(http.MethodPost, "http://example.com", nil)
+				req.AddCookie(&http.Cookie{Name: "foo", Value: "bar"})
+				req.Header.Set("Referer", "http://example.com")
+				return req
+			},
+			expectedResult: "po11cr000000",
+		},
+		{
+			name: "bad accept-language header",
+			request: func() *http.Request {
+				req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil)
+				req.Header.Set("Accept-Language", "aksjdhaslkdhalkjsd")
+				return req
+			},
+			expectedResult: "ge11nn01aksj",
+		},
+		{
+			name: "bad accept-language header 2",
+			request: func() *http.Request {
+				req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil)
+				req.Header.Set("Accept-Language", ",")
+				return req
+			},
+			expectedResult: "ge11nn010000",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := jA4H_a(tt.request())
+			if result != tt.expectedResult {
+				t.Errorf("expected %s, got %s", tt.expectedResult, result)
+			}
+		})
+	}
+}
+
+func TestJA4H_B(t *testing.T) {
+	// This test is only for non-regression
+	// Because go does not keep headers order, we just want to make sure our code always process the headers in the same order
+	tests := []struct {
+		name           string
+		request        func() *http.Request
+		expectedResult string
+	}{
+		{
+			name: "no headers",
+			request: func() *http.Request {
+				req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil)
+				return req
+			},
+			expectedResult: "e3b0c44298fc",
+		},
+		{
+			name: "header with arbitrary content",
+			request: func() *http.Request {
+				req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil)
+				req.Header.Set("X-Custom-Header", "some value")
+				return req
+			},
+			expectedResult: "0a15aba5bbd6",
+		},
+		{
+			name: "header with multiple headers",
+			request: func() *http.Request {
+				req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil)
+				req.Header.Set("X-Custom-Header", "some value")
+				req.Header.Set("Authorization", "Bearer token")
+				return req
+			},
+			expectedResult: "bbfc6cf16ecb",
+		},
+		{
+			name: "curl-like request",
+			request: func() *http.Request {
+				req, _ := http.NewRequest(http.MethodGet, "http://localhost", nil)
+				req.Header.Set("Host", "localhost")
+				req.Header.Set("User-Agent", "curl/8.12.1")
+				req.Header.Set("Accept", "*/*")
+				return req
+			},
+			expectedResult: "4722709a6f34",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := jA4H_b(tt.request())
+			if result != tt.expectedResult {
+				t.Errorf("expected %s, got %s", tt.expectedResult, result)
+			}
+		})
+	}
+}
+
+func TestJA4H_C(t *testing.T) {
+	tests := []struct {
+		name           string
+		cookies        func() []*http.Cookie
+		expectedResult string
+	}{
+		{
+			name: "no cookies",
+			cookies: func() []*http.Cookie {
+				req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil)
+				return req.Cookies()
+			},
+			expectedResult: "000000000000",
+		},
+		{
+			name: "one cookie",
+			cookies: func() []*http.Cookie {
+				req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil)
+				req.AddCookie(&http.Cookie{Name: "foo", Value: "bar"})
+				return req.Cookies()
+			},
+			expectedResult: "2c26b46b68ff",
+		},
+		{
+			name: "duplicate cookies",
+			cookies: func() []*http.Cookie {
+				req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil)
+				req.AddCookie(&http.Cookie{Name: "foo", Value: "bar"})
+				req.AddCookie(&http.Cookie{Name: "foo", Value: "bar2"})
+				return req.Cookies()
+			},
+			expectedResult: "8990ce24137b",
+		},
+		{
+			name: "multiple cookies",
+			cookies: func() []*http.Cookie {
+				req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil)
+				req.AddCookie(&http.Cookie{Name: "foo", Value: "bar"})
+				req.AddCookie(&http.Cookie{Name: "bar", Value: "foo"})
+				cookies := req.Cookies()
+				slices.SortFunc(cookies, func(a, b *http.Cookie) int {
+					return strings.Compare(a.Name, b.Name)
+				})
+				return cookies
+			},
+			expectedResult: "41557db67d60",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := jA4H_c(tt.cookies())
+			if result != tt.expectedResult {
+				t.Errorf("expected %s, got %s", tt.expectedResult, result)
+			}
+		})
+	}
+}
+
+func TestJA4H_D(t *testing.T) {
+	tests := []struct {
+		name           string
+		cookies        func() []*http.Cookie
+		expectedResult string
+	}{
+		{
+			name: "no cookies",
+			cookies: func() []*http.Cookie {
+				req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil)
+				return req.Cookies()
+			},
+			expectedResult: "000000000000",
+		},
+		{
+			name: "one cookie",
+			cookies: func() []*http.Cookie {
+				req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil)
+				req.AddCookie(&http.Cookie{Name: "foo", Value: "bar"})
+				return req.Cookies()
+			},
+			expectedResult: "3ba8907e7a25",
+		},
+		{
+			name: "duplicate cookies",
+			cookies: func() []*http.Cookie {
+				req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil)
+				req.AddCookie(&http.Cookie{Name: "foo", Value: "bar"})
+				req.AddCookie(&http.Cookie{Name: "foo", Value: "bar2"})
+				return req.Cookies()
+			},
+			expectedResult: "975821a3a881",
+		},
+		{
+			name: "multiple cookies",
+			cookies: func() []*http.Cookie {
+				req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil)
+				req.AddCookie(&http.Cookie{Name: "foo", Value: "bar"})
+				req.AddCookie(&http.Cookie{Name: "bar", Value: "foo"})
+				cookies := req.Cookies()
+				slices.SortFunc(cookies, func(a, b *http.Cookie) int {
+					return strings.Compare(a.Name, b.Name)
+				})
+				return cookies
+			},
+			expectedResult: "70f8bee1efb8",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := jA4H_d(tt.cookies())
+			if result != tt.expectedResult {
+				t.Errorf("expected %s, got %s", tt.expectedResult, result)
+			}
+		})
+	}
+}
+
+func TestJA4H(t *testing.T) {
+	tests := []struct {
+		name         string
+		req          func() *http.Request
+		expectedHash string
+	}{
+		{
+			name: "Basic GET - No cookies",
+			req: func() *http.Request {
+				req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil)
+				return req
+			},
+			expectedHash: "ge11nn000000_e3b0c44298fc_000000000000_000000000000",
+		},
+		{
+			name: "Basic GET - With cookies",
+			req: func() *http.Request {
+				req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil)
+				req.AddCookie(&http.Cookie{Name: "session", Value: "12345"})
+				return req
+			},
+			expectedHash: "ge11cn000000_e3b0c44298fc_3f3af1ecebbd_86a3f0069fcd",
+		},
+		{
+			name: "Basic GET - Multiple cookies",
+			req: func() *http.Request {
+				req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil)
+				req.AddCookie(&http.Cookie{Name: "foo", Value: "bar"})
+				req.AddCookie(&http.Cookie{Name: "baz", Value: "qux"})
+				return req
+			},
+			expectedHash: "ge11cn000000_e3b0c44298fc_bd87575d11f6_d401f362552e",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			hash := JA4H(test.req())
+			if hash != test.expectedHash {
+				t.Errorf("expected %s, got %s", test.expectedHash, hash)
+			}
+		})
+	}
+
+}

pkg/appsec/request.go

@@ -16,12 +16,13 @@ import (
 )
 
 const (
-	URIHeaderName       = "X-Crowdsec-Appsec-Uri"
-	VerbHeaderName      = "X-Crowdsec-Appsec-Verb"
-	HostHeaderName      = "X-Crowdsec-Appsec-Host"
-	IPHeaderName        = "X-Crowdsec-Appsec-Ip"
-	APIKeyHeaderName    = "X-Crowdsec-Appsec-Api-Key"
-	UserAgentHeaderName = "X-Crowdsec-Appsec-User-Agent"
+	URIHeaderName         = "X-Crowdsec-Appsec-Uri"
+	VerbHeaderName        = "X-Crowdsec-Appsec-Verb"
+	HostHeaderName        = "X-Crowdsec-Appsec-Host"
+	IPHeaderName          = "X-Crowdsec-Appsec-Ip"
+	APIKeyHeaderName      = "X-Crowdsec-Appsec-Api-Key"
+	UserAgentHeaderName   = "X-Crowdsec-Appsec-User-Agent"
+	HTTPVersionHeaderName = "X-Crowdsec-Appsec-Http-Version"
 )
 
 type ParsedRequest struct {
@@ -313,6 +314,28 @@ func NewParsedRequestFromRequest(r *http.Request, logger *log.Entry) (ParsedRequ
 
 	userAgent := r.Header.Get(UserAgentHeaderName) //This one is optional
 
+	httpVersion := r.Header.Get(HTTPVersionHeaderName)
+	if httpVersion == "" {
+		logger.Debugf("missing '%s' header", HTTPVersionHeaderName)
+	}
+
+	if httpVersion != "" && len(httpVersion) == 2 &&
+		httpVersion[0] >= '0' && httpVersion[0] <= '9' &&
+		httpVersion[1] >= '0' && httpVersion[1] <= '9' {
+		major := httpVersion[0]
+		minor := httpVersion[1]
+
+		r.ProtoMajor = int(major - '0')
+		r.ProtoMinor = int(minor - '0')
+		if r.ProtoMajor == 2 && r.ProtoMinor == 0 {
+			r.Proto = "HTTP/2"
+		} else {
+			r.Proto = "HTTP/" + string(major) + "." + string(minor)
+		}
+	} else {
+		logger.Warnf("Invalid value %s for HTTP version header", httpVersion)
+	}
+
 	// delete those headers before coraza process the request
 	delete(r.Header, IPHeaderName)
 	delete(r.Header, HostHeaderName)
@@ -320,6 +343,7 @@ func NewParsedRequestFromRequest(r *http.Request, logger *log.Entry) (ParsedRequ
 	delete(r.Header, VerbHeaderName)
 	delete(r.Header, UserAgentHeaderName)
 	delete(r.Header, APIKeyHeaderName)
+	delete(r.Header, HTTPVersionHeaderName)
 
 	originalHTTPRequest := r.Clone(r.Context())
 	originalHTTPRequest.Body = io.NopCloser(bytes.NewBuffer(body))

pkg/exprhelpers/expr_lib.go

@@ -2,6 +2,7 @@ package exprhelpers
 
 import (
 	"net"
+	"net/http"
 	"time"
 
 	"github.com/oschwald/geoip2-golang"
@@ -493,6 +494,13 @@ var exprFuncs = []exprCustomFunc{
 			new(func(string) *net.IPNet),
 		},
 	},
+	{
+		name:     "JA4H",
+		function: JA4H,
+		signature: []interface{}{
+			new(func(*http.Request) string),
+		},
+	},
 }
 
 //go 1.20 "CutPrefix":              strings.CutPrefix,

pkg/exprhelpers/waf.go

@@ -0,0 +1,13 @@
+package exprhelpers
+
+import (
+	"net/http"
+
+	"github.com/crowdsecurity/crowdsec/pkg/appsec/ja4h"
+)
+
+// JA4H(req *http.Request) string
+func JA4H(params ...any) (any, error) {
+	req := params[0].(*http.Request)
+	return ja4h.JA4H(req), nil
+}

pkg/exprhelpers/waf_test.go

@@ -0,0 +1,78 @@
+package exprhelpers
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestJA4H(t *testing.T) {
+
+	tests := []struct {
+		name         string
+		method       string
+		url          string
+		cookies      map[string]string
+		headers      map[string]string
+		expectedHash string
+	}{
+		{
+			name:         "Basic GET - No cookies",
+			method:       "GET",
+			url:          "http://example.com",
+			cookies:      map[string]string{},
+			headers:      map[string]string{},
+			expectedHash: "ge11nn000000_e3b0c44298fc_000000000000_000000000000",
+		},
+		{
+			name:         "Basic POST - No cookies",
+			method:       "POST",
+			url:          "http://example.com",
+			cookies:      map[string]string{},
+			headers:      map[string]string{},
+			expectedHash: "po11nn000000_e3b0c44298fc_000000000000_000000000000",
+		},
+		{
+			name:   "GET - With cookies",
+			method: "GET",
+			url:    "http://example.com/foobar",
+			cookies: map[string]string{
+				"foo": "bar",
+				"baz": "qux",
+			},
+			headers: map[string]string{
+				"User-Agent": "Mozilla/5.0",
+			},
+			expectedHash: "ge11cn010000_b8bcd45ac095_bd87575d11f6_d401f362552e",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			req, err := http.NewRequest(test.method, test.url, nil)
+			if err != nil {
+				t.Fatalf("Failed to create request: %s", err)
+			}
+
+			for key, value := range test.cookies {
+				req.AddCookie(&http.Cookie{
+					Name:  key,
+					Value: value,
+				})
+			}
+
+			for key, value := range test.headers {
+				req.Header.Add(key, value)
+			}
+
+			hash, err := JA4H(req)
+			require.NoError(t, err)
+
+			if hash != test.expectedHash {
+				t.Fatalf("JA4H returned unexpected hash: %s", hash)
+			}
+		})
+	}
+
+}