chore: improve test_timeout robustness (#4494)

1. use assert_eventually
2. add more logs
3. unrelated - add a stats event to track timeout shutdowns.
This commit is contained in:
Roman Gershman 2025-01-22 14:24:29 +02:00 committed by GitHub
parent 4a2f2e3d04
commit 451da72c41
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 16 additions and 4 deletions

View file

@ -2290,6 +2290,7 @@ void ServerFamily::Info(CmdArgList args, const CommandContext& cmd_cntx) {
append("pipeline_queue_length", m.facade_stats.conn_stats.dispatch_queue_entries);
append("send_delay_ms", GetDelayMs(m.oldest_pending_send_ts));
append("timeout_disconnects", m.coordinator_stats.conn_timeout_events);
}
if (should_enter("MEMORY")) {

View file

@ -34,7 +34,7 @@ ServerState::Stats::Stats(unsigned num_shards) : tx_width_freq_arr(num_shards) {
}
ServerState::Stats& ServerState::Stats::Add(const ServerState::Stats& other) {
static_assert(sizeof(Stats) == 19 * 8, "Stats size mismatch");
static_assert(sizeof(Stats) == 20 * 8, "Stats size mismatch");
#define ADD(x) this->x += (other.x)
@ -61,7 +61,7 @@ ServerState::Stats& ServerState::Stats::Add(const ServerState::Stats& other) {
ADD(compressed_blobs);
ADD(oom_error_cmd_cnt);
ADD(conn_timeout_events);
if (this->tx_width_freq_arr.size() > 0) {
DCHECK_EQ(this->tx_width_freq_arr.size(), other.tx_width_freq_arr.size());
this->tx_width_freq_arr += other.tx_width_freq_arr;
@ -279,6 +279,7 @@ void ServerState::ConnectionsWatcherFb(util::ListenerInterface* main) {
if (conn) {
VLOG(1) << "Closing connection due to timeout: " << conn->GetClientInfo();
conn->ShutdownSelf();
stats.conn_timeout_events++;
}
}
}

View file

@ -131,6 +131,7 @@ class ServerState { // public struct - to allow initialization.
// Number of times we rejected command dispatch due to OOM condition.
uint64_t oom_error_cmd_cnt = 0;
uint32_t conn_timeout_events = 0;
std::valarray<uint64_t> tx_width_freq_arr;
};