feat(tools/replay): Add pipeline latency distribution data (#4990)

feat(replay): add latency distributions

* also add avg latency
* also include pipeline range
* display both at the end

tools/replay/go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/gookit/color v1.4.2 // indirect
+	github.com/influxdata/tdigest v0.0.1 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect

tools/replay/go.sum

@@ -9,8 +9,11 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/gookit/color v1.4.2 h1:tXy44JFSFkKnELV6WaMo/lLfu/meqITX3iAV52do7lk=
 github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ=
+github.com/influxdata/tdigest v0.0.1 h1:XpFptwYmnEKUqmkcDjrzffswZ3nvNeevbUSLPP/ZzIY=
+github.com/influxdata/tdigest v0.0.1/go.mod h1:Z0kXnxzbTC2qrx4NaIzYkE1k66+6oEDQTvL95hQFh5Y=
 github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -27,11 +30,15 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
 github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
+golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44 h1:Bli41pIlzTzf3KEY06n+xnzK/BESIg2ze4Pgfh/aI8c=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
+gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

tools/replay/main.go

@@ -19,20 +19,62 @@ var fPace = flag.Bool("pace", true, "whether to pace the traffic according to th
 var fSkip = flag.Uint("skip", 0, "skip N records")
 
 func RenderTable(area *pterm.AreaPrinter, files []string, workers []FileWorker) {
-	tableData := pterm.TableData{{"file", "parsed", "processed", "delayed", "clients"}}
+	tableData := pterm.TableData{{"file", "parsed", "processed", "delayed", "clients", "avg(us)", "p50(us)", "p75(us)", "p90(us)", "p99(us)"}}
 	for i := range workers {
+		workers[i].latencyMu.Lock()
+		avg := 0.0
+		if workers[i].latencyCount > 0 {
+			avg = workers[i].latencySum / float64(workers[i].latencyCount)
+		}
+		p50 := workers[i].latencyDigest.Quantile(0.5)
+		p75 := workers[i].latencyDigest.Quantile(0.75)
+		p90 := workers[i].latencyDigest.Quantile(0.9)
+		p99 := workers[i].latencyDigest.Quantile(0.99)
+		workers[i].latencyMu.Unlock()
 		tableData = append(tableData, []string{
 			files[i],
 			fmt.Sprint(atomic.LoadUint64(&workers[i].parsed)),
 			fmt.Sprint(atomic.LoadUint64(&workers[i].processed)),
 			fmt.Sprint(atomic.LoadUint64(&workers[i].delayed)),
 			fmt.Sprint(atomic.LoadUint64(&workers[i].clients)),
+			fmt.Sprintf("%.0f", avg),
+			fmt.Sprintf("%.0f", p50),
+			fmt.Sprintf("%.0f", p75),
+			fmt.Sprintf("%.0f", p90),
+			fmt.Sprintf("%.0f", p99),
 		})
 	}
 	content, _ := pterm.DefaultTable.WithHasHeader().WithBoxed().WithData(tableData).Srender()
 	area.Update(content)
 }
 
+// RenderPipelineRangesTable renders the latency digests for each pipeline range
+func RenderPipelineRangesTable(area *pterm.AreaPrinter, files []string, workers []FileWorker) {
+	tableData := pterm.TableData{{"file", "Pipeline Range", "p50(us)", "p75(us)", "p90(us)", "p99(us)"}}
+	for i := range workers {
+		workers[i].latencyMu.Lock()
+		for _, rng := range pipelineRanges {
+			if digest, ok := workers[i].perRange[rng.label]; ok {
+				p50 := digest.Quantile(0.5)
+				p75 := digest.Quantile(0.75)
+				p90 := digest.Quantile(0.9)
+				p99 := digest.Quantile(0.99)
+				tableData = append(tableData, []string{
+					files[i],
+					rng.label,
+					fmt.Sprintf("%.0f", p50),
+					fmt.Sprintf("%.0f", p75),
+					fmt.Sprintf("%.0f", p90),
+					fmt.Sprintf("%.0f", p99),
+				})
+			}
+		}
+		workers[i].latencyMu.Unlock()
+	}
+	content, _ := pterm.DefaultTable.WithHasHeader().WithBoxed().WithData(tableData).Srender()
+	area.Update(content)
+}
+
 func Run(files []string) {
 	timeOffset := time.Now().Add(500 * time.Millisecond).Sub(DetermineBaseTime(files))
 	fmt.Println("Offset -> ", timeOffset)
@@ -64,6 +106,8 @@ func Run(files []string) {
 	}
 
 	RenderTable(area, files, workers) // to show last stats
+	areaPipelineRanges, _ := pterm.DefaultArea.WithCenter().Start()
+	RenderPipelineRangesTable(areaPipelineRanges, files, workers) // to render per pipeline-range latency digests
 }
 
 func Print(files []string) {

tools/replay/workers.go

@@ -10,6 +10,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/influxdata/tdigest"
 	"github.com/redis/go-redis/v9"
 )
 
@@ -47,6 +48,18 @@ type ClientWorker struct {
 	pipe      redis.Pipeliner
 }
 
+// Pipeline length ranges for summary
+var pipelineRanges = []struct {
+	label string
+	min   int
+	max   int // inclusive, except last
+}{
+	{"0-29", 0, 29},
+	{"30-79", 30, 79},
+	{"80-199", 80, 199},
+	{"200+", 200, 1 << 30},
+}
+
 // Handles a single file and distributes messages to clients
 type FileWorker struct {
 	clientGroup sync.WaitGroup
@@ -56,6 +69,33 @@ type FileWorker struct {
 	delayed   uint64
 	parsed    uint64
 	clients   uint64
+
+	latencyDigest *tdigest.TDigest
+	latencyMu     sync.Mutex
+
+	latencySum   float64 // sum of all batch latencies (microseconds)
+	latencyCount uint64  // number of batches
+
+	// per-pipeline-range latency digests
+	perRange map[string]*tdigest.TDigest
+}
+
+// Helper function to track latency and update digests
+func trackLatency(worker *FileWorker, batchLatency float64, size int) {
+	worker.latencyMu.Lock()
+	defer worker.latencyMu.Unlock()
+	worker.latencyDigest.Add(batchLatency, 1)
+	worker.latencySum += batchLatency
+	worker.latencyCount++
+	// Add to per-range digest
+	if worker.perRange != nil {
+		for _, rng := range pipelineRanges {
+			if size >= rng.min && size <= rng.max {
+				worker.perRange[rng.label].Add(batchLatency, 1)
+				break
+			}
+		}
+	}
 }
 
 func (c *ClientWorker) Run(pace bool, worker *FileWorker) {
@@ -79,13 +119,19 @@ func (c *ClientWorker) Run(pace bool, worker *FileWorker) {
 
 		if msg.HasMore == 0 {
 			size := c.pipe.Len()
+			start := time.Now()
 			c.pipe.Exec(context.Background())
+			batchLatency := float64(time.Since(start).Microseconds())
+			trackLatency(worker, batchLatency, size)
 			c.processed += uint(size)
 		}
 	}
 
 	if size := c.pipe.Len(); size >= 0 {
+		start := time.Now()
 		c.pipe.Exec(context.Background())
+		batchLatency := float64(time.Since(start).Microseconds())
+		trackLatency(worker, batchLatency, size)
 		c.processed += uint(size)
 	}
 
@@ -106,6 +152,11 @@ func NewClient(w *FileWorker, pace bool) *ClientWorker {
 }
 
 func (w *FileWorker) Run(file string, wg *sync.WaitGroup) {
+	w.latencyDigest = tdigest.NewWithCompression(1000)
+	w.perRange = make(map[string]*tdigest.TDigest)
+	for _, rng := range pipelineRanges {
+		w.perRange[rng.label] = tdigest.NewWithCompression(500)
+	}
 	clients := make(map[uint32]*ClientWorker, 0)
 	recordId := uint64(0)
 	err := parseRecords(file, func(r Record) bool {