wineventlog: add support for replaying evtx files (#3278)

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/appleboy/gin-jwt/v2 v2.9.2
 	github.com/aws/aws-lambda-go v1.47.0
 	github.com/aws/aws-sdk-go v1.52.0
-	github.com/beevik/etree v1.3.0
+	github.com/beevik/etree v1.4.1
 	github.com/blackfireio/osinfo v1.0.5
 	github.com/bluele/gcache v0.0.2
 	github.com/buger/jsonparser v1.1.1

go.sum

@@ -58,6 +58,8 @@ github.com/aws/aws-sdk-go v1.52.0 h1:ptgek/4B2v/ljsjYSEvLQ8LTD+SQyrqhOOWvHc/VGPI
 github.com/aws/aws-sdk-go v1.52.0/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/beevik/etree v1.3.0 h1:hQTc+pylzIKDb23yYprodCWWTt+ojFfUZyzU09a/hmU=
 github.com/beevik/etree v1.3.0/go.mod h1:aiPf89g/1k3AShMVAzriilpcE4R/Vuor90y83zVZWFc=
+github.com/beevik/etree v1.4.1 h1:PmQJDDYahBGNKDcpdX8uPy1xRCwoCGVUiW669MEirVI=
+github.com/beevik/etree v1.4.1/go.mod h1:gPNJNaBGVZ9AwsidazFZyygnd+0pAU38N4D+WemwKNs=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=

pkg/acquisition/modules/wineventlog/wineventlog_windows.go

@@ -5,7 +5,9 @@ import (
 	"encoding/xml"
 	"errors"
 	"fmt"
+	"net/url"
 	"runtime"
+	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -30,7 +32,7 @@ type WinEventLogConfiguration struct {
 	EventLevel                        string `yaml:"event_level"`
 	EventIDs                          []int  `yaml:"event_ids"`
 	XPathQuery                        string `yaml:"xpath_query"`
-	EventFile                         string `yaml:"event_file"`
+	EventFile                         string
 	PrettyName                        string `yaml:"pretty_name"`
 }
 
@@ -48,10 +50,13 @@ type QueryList struct {
 }
 
 type Select struct {
-	Path  string `xml:"Path,attr"`
+	Path  string `xml:"Path,attr,omitempty"`
 	Query string `xml:",chardata"`
 }
 
+// 0 identifies the local machine in windows APIs
+const localMachine = 0
+
 var linesRead = prometheus.NewCounterVec(
 	prometheus.CounterOpts{
 		Name: "cs_winevtlogsource_hits_total",
@@ -212,20 +217,28 @@ func (w *WinEventLogSource) getEvents(out chan types.Event, t *tomb.Tomb) error
 	}
 }
 
-func (w *WinEventLogSource) generateConfig(query string) (*winlog.SubscribeConfig, error) {
+func (w *WinEventLogSource) generateConfig(query string, live bool) (*winlog.SubscribeConfig, error) {
 	var config winlog.SubscribeConfig
 	var err error
 
-	// Create a subscription signaler.
-	config.SignalEvent, err = windows.CreateEvent(
-		nil, // Default security descriptor.
-		1,   // Manual reset.
-		1,   // Initial state is signaled.
-		nil) // Optional name.
-	if err != nil {
-		return &config, fmt.Errorf("windows.CreateEvent failed: %v", err)
+	if live {
+		// Create a subscription signaler.
+		config.SignalEvent, err = windows.CreateEvent(
+			nil, // Default security descriptor.
+			1,   // Manual reset.
+			1,   // Initial state is signaled.
+			nil) // Optional name.
+		if err != nil {
+			return &config, fmt.Errorf("windows.CreateEvent failed: %v", err)
+		}
+		config.Flags = wevtapi.EvtSubscribeToFutureEvents
+	} else {
+		config.ChannelPath, err = syscall.UTF16PtrFromString(w.config.EventFile)
+		if err != nil {
+			return &config, fmt.Errorf("syscall.UTF16PtrFromString failed: %v", err)
+		}
+		config.Flags = wevtapi.EvtQueryFilePath | wevtapi.EvtQueryForwardDirection
 	}
-	config.Flags = wevtapi.EvtSubscribeToFutureEvents
 	config.Query, err = syscall.UTF16PtrFromString(query)
 	if err != nil {
 		return &config, fmt.Errorf("syscall.UTF16PtrFromString failed: %v", err)
@@ -283,7 +296,7 @@ func (w *WinEventLogSource) Configure(yamlConfig []byte, logger *log.Entry, Metr
 		return err
 	}
 
-	w.evtConfig, err = w.generateConfig(w.query)
+	w.evtConfig, err = w.generateConfig(w.query, true)
 	if err != nil {
 		return err
 	}
@@ -292,6 +305,78 @@ func (w *WinEventLogSource) Configure(yamlConfig []byte, logger *log.Entry, Metr
 }
 
 func (w *WinEventLogSource) ConfigureByDSN(dsn string, labels map[string]string, logger *log.Entry, uuid string) error {
+	if !strings.HasPrefix(dsn, "wineventlog://") {
+		return fmt.Errorf("invalid DSN %s for wineventlog source, must start with wineventlog://", dsn)
+	}
+
+	w.logger = logger
+	w.config = WinEventLogConfiguration{}
+
+	dsn = strings.TrimPrefix(dsn, "wineventlog://")
+
+	args := strings.Split(dsn, "?")
+
+	if args[0] == "" {
+		return errors.New("empty wineventlog:// DSN")
+	}
+
+	if len(args) > 2 {
+		return errors.New("too many arguments in DSN")
+	}
+
+	w.config.EventFile = args[0]
+
+	if len(args) == 2 && args[1] != "" {
+		params, err := url.ParseQuery(args[1])
+		if err != nil {
+			return fmt.Errorf("failed to parse DSN parameters: %w", err)
+		}
+
+		for key, value := range params {
+			switch key {
+			case "log_level":
+				if len(value) != 1 {
+					return errors.New("log_level must be a single value")
+				}
+				lvl, err := log.ParseLevel(value[0])
+				if err != nil {
+					return fmt.Errorf("failed to parse log_level: %s", err)
+				}
+				w.logger.Logger.SetLevel(lvl)
+			case "event_id":
+				for _, id := range value {
+					evtid, err := strconv.Atoi(id)
+					if err != nil {
+						return fmt.Errorf("failed to parse event_id: %s", err)
+					}
+					w.config.EventIDs = append(w.config.EventIDs, evtid)
+				}
+			case "event_level":
+				if len(value) != 1 {
+					return errors.New("event_level must be a single value")
+				}
+				w.config.EventLevel = value[0]
+			}
+		}
+	}
+
+	var err error
+
+	//FIXME: handle custom xpath query
+	w.query, err = w.buildXpathQuery()
+
+	if err != nil {
+		return fmt.Errorf("buildXpathQuery failed: %w", err)
+	}
+
+	w.logger.Debugf("query: %s\n", w.query)
+
+	w.evtConfig, err = w.generateConfig(w.query, false)
+
+	if err != nil {
+		return fmt.Errorf("generateConfig failed: %w", err)
+	}
+
 	return nil
 }
 
@@ -300,10 +385,57 @@ func (w *WinEventLogSource) GetMode() string {
 }
 
 func (w *WinEventLogSource) SupportedModes() []string {
-	return []string{configuration.TAIL_MODE}
+	return []string{configuration.TAIL_MODE, configuration.CAT_MODE}
 }
 
 func (w *WinEventLogSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error {
+
+	handle, err := wevtapi.EvtQuery(localMachine, w.evtConfig.ChannelPath, w.evtConfig.Query, w.evtConfig.Flags)
+
+	if err != nil {
+		return fmt.Errorf("EvtQuery failed: %v", err)
+	}
+
+	defer winlog.Close(handle)
+
+	publisherCache := make(map[string]windows.Handle)
+	defer func() {
+		for _, h := range publisherCache {
+			winlog.Close(h)
+		}
+	}()
+
+OUTER_LOOP:
+	for {
+		select {
+		case <-t.Dying():
+			w.logger.Infof("wineventlog is dying")
+			return nil
+		default:
+			evts, err := w.getXMLEvents(w.evtConfig, publisherCache, handle, 500)
+			if err == windows.ERROR_NO_MORE_ITEMS {
+				log.Info("No more items")
+				break OUTER_LOOP
+			} else if err != nil {
+				return fmt.Errorf("getXMLEvents failed: %v", err)
+			}
+			w.logger.Debugf("Got %d events", len(evts))
+			for _, evt := range evts {
+				w.logger.Tracef("Event: %s", evt)
+				if w.metricsLevel != configuration.METRICS_NONE {
+					linesRead.With(prometheus.Labels{"source": w.name}).Inc()
+				}
+				l := types.Line{}
+				l.Raw = evt
+				l.Module = w.GetName()
+				l.Labels = w.config.Labels
+				l.Time = time.Now()
+				l.Src = w.name
+				l.Process = true
+				out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE}
+			}
+		}
+	}
 	return nil
 }
 

pkg/acquisition/modules/wineventlog/wineventlog_windows_test.go

@@ -4,7 +4,6 @@ package wineventlogacquisition
 
 import (
 	"context"
-	"runtime"
 	"testing"
 	"time"
 
@@ -19,9 +18,8 @@ import (
 )
 
 func TestBadConfiguration(t *testing.T) {
-	if runtime.GOOS != "windows" {
-		t.Skip("Skipping test on non-windows OS")
-	}
+	exprhelpers.Init(nil)
+
 	tests := []struct {
 		config      string
 		expectedErr string
@@ -64,9 +62,8 @@ xpath_query: test`,
 }
 
 func TestQueryBuilder(t *testing.T) {
-	if runtime.GOOS != "windows" {
-		t.Skip("Skipping test on non-windows OS")
-	}
+	exprhelpers.Init(nil)
+
 	tests := []struct {
 		config        string
 		expectedQuery string
@@ -130,10 +127,8 @@ event_level: bla`,
 }
 
 func TestLiveAcquisition(t *testing.T) {
+	exprhelpers.Init(nil)
 	ctx := context.Background()
-	if runtime.GOOS != "windows" {
-		t.Skip("Skipping test on non-windows OS")
-	}
 
 	tests := []struct {
 		config        string
@@ -227,3 +222,82 @@ event_ids:
 		to.Wait()
 	}
 }
+
+func TestOneShotAcquisition(t *testing.T) {
+	tests := []struct {
+		name                 string
+		dsn                  string
+		expectedCount        int
+		expectedErr          string
+		expectedConfigureErr string
+	}{
+		{
+			name:          "non-existing file",
+			dsn:           `wineventlog://foo.evtx`,
+			expectedCount: 0,
+			expectedErr:   "The system cannot find the file specified.",
+		},
+		{
+			name:                 "empty DSN",
+			dsn:                  `wineventlog://`,
+			expectedCount:        0,
+			expectedConfigureErr: "empty wineventlog:// DSN",
+		},
+		{
+			name:          "existing file",
+			dsn:           `wineventlog://test_files/Setup.evtx`,
+			expectedCount: 24,
+			expectedErr:   "",
+		},
+		{
+			name:          "filter on event_id",
+			dsn:           `wineventlog://test_files/Setup.evtx?event_id=2`,
+			expectedCount: 1,
+		},
+		{
+			name:          "filter on event_id",
+			dsn:           `wineventlog://test_files/Setup.evtx?event_id=2&event_id=3`,
+			expectedCount: 24,
+		},
+	}
+
+	exprhelpers.Init(nil)
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			lineCount := 0
+			to := &tomb.Tomb{}
+			c := make(chan types.Event)
+			f := WinEventLogSource{}
+			err := f.ConfigureByDSN(test.dsn, map[string]string{"type": "wineventlog"}, log.WithField("type", "windowseventlog"), "")
+
+			if test.expectedConfigureErr != "" {
+				assert.Contains(t, err.Error(), test.expectedConfigureErr)
+				return
+			}
+
+			require.NoError(t, err)
+
+			go func() {
+				for {
+					select {
+					case <-c:
+						lineCount++
+					case <-to.Dying():
+						return
+					}
+				}
+			}()
+
+			err = f.OneShotAcquisition(c, to)
+			if test.expectedErr != "" {
+				assert.Contains(t, err.Error(), test.expectedErr)
+			} else {
+				require.NoError(t, err)
+
+				time.Sleep(2 * time.Second)
+				assert.Equal(t, test.expectedCount, lineCount)
+			}
+		})
+	}
+}

pkg/exprhelpers/helpers.go

@@ -129,7 +129,7 @@ func Init(databaseClient *database.Client) error {
 	dataFileRegex = make(map[string][]*regexp.Regexp)
 	dataFileRe2 = make(map[string][]*re2.Regexp)
 	dbClient = databaseClient
-
+	XMLCacheInit()
 	return nil
 }
 

pkg/exprhelpers/xml.go

@@ -1,43 +1,103 @@
 package exprhelpers
 
 import (
+	"errors"
+	"sync"
+	"time"
+
 	"github.com/beevik/etree"
+	"github.com/bluele/gcache"
+	"github.com/cespare/xxhash/v2"
 	log "github.com/sirupsen/logrus"
 )
 
-var pathCache = make(map[string]etree.Path)
+var (
+	pathCache        = make(map[string]etree.Path)
+	rwMutex          = sync.RWMutex{}
+	xmlDocumentCache gcache.Cache
+)
+
+func compileOrGetPath(path string) (etree.Path, error) {
+	rwMutex.RLock()
+	compiledPath, ok := pathCache[path]
+	rwMutex.RUnlock()
+
+	if !ok {
+		var err error
+		compiledPath, err = etree.CompilePath(path)
+		if err != nil {
+			return etree.Path{}, err
+		}
+
+		rwMutex.Lock()
+		pathCache[path] = compiledPath
+		rwMutex.Unlock()
+	}
+
+	return compiledPath, nil
+}
+
+func getXMLDocumentFromCache(xmlString string) (*etree.Document, error) {
+	cacheKey := xxhash.Sum64String(xmlString)
+	cacheObj, err := xmlDocumentCache.Get(cacheKey)
+
+	if err != nil && !errors.Is(err, gcache.KeyNotFoundError) {
+		return nil, err
+	}
+
+	doc, ok := cacheObj.(*etree.Document)
+	if !ok || cacheObj == nil {
+		doc = etree.NewDocument()
+		if err := doc.ReadFromString(xmlString); err != nil {
+			return nil, err
+		}
+		if err := xmlDocumentCache.Set(cacheKey, doc); err != nil {
+			log.Warnf("Could not set XML document in cache: %s", err)
+		}
+	}
+
+	return doc, nil
+}
+
+func XMLCacheInit() {
+	gc := gcache.New(50)
+	// 	Short cache expiration because we each line we read is different, but we can call multiple times XML helpers on each of them
+	gc.Expiration(5 * time.Second)
+	gc = gc.LRU()
+
+	xmlDocumentCache = gc.Build()
+}
 
 // func XMLGetAttributeValue(xmlString string, path string, attributeName string) string {
 func XMLGetAttributeValue(params ...any) (any, error) {
 	xmlString := params[0].(string)
 	path := params[1].(string)
 	attributeName := params[2].(string)
-	if _, ok := pathCache[path]; !ok {
-		compiledPath, err := etree.CompilePath(path)
-		if err != nil {
-			log.Errorf("Could not compile path %s: %s", path, err)
-			return "", nil
-		}
-		pathCache[path] = compiledPath
+
+	compiledPath, err := compileOrGetPath(path)
+	if err != nil {
+		log.Errorf("Could not compile path %s: %s", path, err)
+		return "", nil
 	}
 
-	compiledPath := pathCache[path]
-	doc := etree.NewDocument()
-	err := doc.ReadFromString(xmlString)
+	doc, err := getXMLDocumentFromCache(xmlString)
 	if err != nil {
 		log.Tracef("Could not parse XML: %s", err)
 		return "", nil
 	}
+
 	elem := doc.FindElementPath(compiledPath)
 	if elem == nil {
 		log.Debugf("Could not find element %s", path)
 		return "", nil
 	}
+
 	attr := elem.SelectAttr(attributeName)
 	if attr == nil {
 		log.Debugf("Could not find attribute %s", attributeName)
 		return "", nil
 	}
+
 	return attr.Value, nil
 }
 
@@ -45,26 +105,24 @@ func XMLGetAttributeValue(params ...any) (any, error) {
 func XMLGetNodeValue(params ...any) (any, error) {
 	xmlString := params[0].(string)
 	path := params[1].(string)
-	if _, ok := pathCache[path]; !ok {
-		compiledPath, err := etree.CompilePath(path)
-		if err != nil {
-			log.Errorf("Could not compile path %s: %s", path, err)
-			return "", nil
-		}
-		pathCache[path] = compiledPath
+
+	compiledPath, err := compileOrGetPath(path)
+	if err != nil {
+		log.Errorf("Could not compile path %s: %s", path, err)
+		return "", nil
 	}
 
-	compiledPath := pathCache[path]
-	doc := etree.NewDocument()
-	err := doc.ReadFromString(xmlString)
+	doc, err := getXMLDocumentFromCache(xmlString)
 	if err != nil {
 		log.Tracef("Could not parse XML: %s", err)
 		return "", nil
 	}
+
 	elem := doc.FindElementPath(compiledPath)
 	if elem == nil {
 		log.Debugf("Could not find element %s", path)
 		return "", nil
 	}
+
 	return elem.Text(), nil
 }