mirrors/dragonfly
1
0
Fork 0
mirror of https://github.com/dragonflydb/dragonfly.git synced 2025-05-10 18:05:44 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

chore: Add stats print for slot migrations (#4456)

Browse source 
* chore: Add stats print for slot migrations

Fixes #4415
This commit is contained in:
Shahar Mike 2025-01-15 13:06:09 +02:00 • committed by GitHub
parent 0eff6c93f2
commit 5ba608b58d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 116 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

45
src/server/journal/cmd_serializer.cc
View file

@ -26,13 +26,18 @@ class CommandAggregator {
}
enum class CommitMode { kAuto, kNoCommit };
void AddArg(string arg, CommitMode commit_mode = CommitMode::kAuto) {
// Returns whether CommitPending() was called
bool AddArg(string arg, CommitMode commit_mode = CommitMode::kAuto) {
agg_bytes_ += arg.size();
members_.push_back(std::move(arg));
if (commit_mode != CommitMode::kNoCommit && agg_bytes_ >= max_aggragation_bytes_) {
CommitPending();
return true;
}
return false;
}
private:
@ -65,26 +70,27 @@ CmdSerializer::CmdSerializer(FlushSerialized cb, size_t max_serialization_buffer
: cb_(std::move(cb)), max_serialization_buffer_size_(max_serialization_buffer_size) {
}
void CmdSerializer::SerializeEntry(string_view key, const PrimeValue& pk, const PrimeValue& pv,
uint64_t expire_ms) {
size_t CmdSerializer::SerializeEntry(string_view key, const PrimeValue& pk, const PrimeValue& pv,
uint64_t expire_ms) {
// We send RESTORE commands for small objects, or objects we don't support breaking.
bool use_restore_serialization = true;
size_t commands = 1;
if (max_serialization_buffer_size_ > 0 && pv.MallocUsed() > max_serialization_buffer_size_) {
switch (pv.ObjType()) {
case OBJ_SET:
SerializeSet(key, pv);
commands = SerializeSet(key, pv);
use_restore_serialization = false;
break;
case OBJ_ZSET:
SerializeZSet(key, pv);
commands = SerializeZSet(key, pv);
use_restore_serialization = false;
break;
case OBJ_HASH:
SerializeHash(key, pv);
commands = SerializeHash(key, pv);
use_restore_serialization = false;
break;
case OBJ_LIST:
SerializeList(key, pv);
commands = SerializeList(key, pv);
use_restore_serialization = false;
break;
case OBJ_STRING:
@ -105,6 +111,7 @@ void CmdSerializer::SerializeEntry(string_view key, const PrimeValue& pk, const
SerializeStickIfNeeded(key, pk);
SerializeExpireIfNeeded(key, expire_ms);
}
return commands;
}
void CmdSerializer::SerializeCommand(string_view cmd, absl::Span<const string_view> args) {
@ -139,54 +146,62 @@ void CmdSerializer::SerializeExpireIfNeeded(string_view key, uint64_t expire_ms)
SerializeCommand("PEXIRE", {key, absl::StrCat(expire_ms)});
}
void CmdSerializer::SerializeSet(string_view key, const PrimeValue& pv) {
size_t CmdSerializer::SerializeSet(string_view key, const PrimeValue& pv) {
CommandAggregator aggregator(
key, [&](absl::Span<const string_view> args) { SerializeCommand("SADD", args); },
max_serialization_buffer_size_);
size_t commands = 0;
container_utils::IterateSet(pv, [&](container_utils::ContainerEntry ce) {
aggregator.AddArg(ce.ToString());
commands += aggregator.AddArg(ce.ToString());
return true;
});
return commands;
}
void CmdSerializer::SerializeZSet(string_view key, const PrimeValue& pv) {
size_t CmdSerializer::SerializeZSet(string_view key, const PrimeValue& pv) {
CommandAggregator aggregator(
key, [&](absl::Span<const string_view> args) { SerializeCommand("ZADD", args); },
max_serialization_buffer_size_);
size_t commands = 0;
container_utils::IterateSortedSet(
pv.GetRobjWrapper(),
[&](container_utils::ContainerEntry ce, double score) {
aggregator.AddArg(absl::StrCat(score), CommandAggregator::CommitMode::kNoCommit);
aggregator.AddArg(ce.ToString());
commands += aggregator.AddArg(ce.ToString());
return true;
},
/*start=*/0, /*end=*/-1, /*reverse=*/false, /*use_score=*/true);
return commands;
}
void CmdSerializer::SerializeHash(string_view key, const PrimeValue& pv) {
size_t CmdSerializer::SerializeHash(string_view key, const PrimeValue& pv) {
CommandAggregator aggregator(
key, [&](absl::Span<const string_view> args) { SerializeCommand("HSET", args); },
max_serialization_buffer_size_);
size_t commands = 0;
container_utils::IterateMap(
pv, [&](container_utils::ContainerEntry k, container_utils::ContainerEntry v) {
aggregator.AddArg(k.ToString(), CommandAggregator::CommitMode::kNoCommit);
aggregator.AddArg(v.ToString());
commands += aggregator.AddArg(v.ToString());
return true;
});
return commands;
}
void CmdSerializer::SerializeList(string_view key, const PrimeValue& pv) {
size_t CmdSerializer::SerializeList(string_view key, const PrimeValue& pv) {
CommandAggregator aggregator(
key, [&](absl::Span<const string_view> args) { SerializeCommand("RPUSH", args); },
max_serialization_buffer_size_);
size_t commands = 0;
container_utils::IterateList(pv, [&](container_utils::ContainerEntry ce) {
aggregator.AddArg(ce.ToString());
commands += aggregator.AddArg(ce.ToString());
return true;
});
return commands;
}
void CmdSerializer::SerializeRestore(string_view key, const PrimeValue& pk, const PrimeValue& pv,

13
src/server/journal/cmd_serializer.h
View file

@ -23,18 +23,19 @@ class CmdSerializer {
explicit CmdSerializer(FlushSerialized cb, size_t max_serialization_buffer_size);
void SerializeEntry(std::string_view key, const PrimeValue& pk, const PrimeValue& pv,
uint64_t expire_ms);
// Returns how many commands we broke this entry into (like multiple HSETs etc)
size_t SerializeEntry(std::string_view key, const PrimeValue& pk, const PrimeValue& pv,
uint64_t expire_ms);
private:
void SerializeCommand(std::string_view cmd, absl::Span<const std::string_view> args);
void SerializeStickIfNeeded(std::string_view key, const PrimeValue& pk);
void SerializeExpireIfNeeded(std::string_view key, uint64_t expire_ms);
void SerializeSet(std::string_view key, const PrimeValue& pv);
void SerializeZSet(std::string_view key, const PrimeValue& pv);
void SerializeHash(std::string_view key, const PrimeValue& pv);
void SerializeList(std::string_view key, const PrimeValue& pv);
size_t SerializeSet(std::string_view key, const PrimeValue& pv);
size_t SerializeZSet(std::string_view key, const PrimeValue& pv);
size_t SerializeHash(std::string_view key, const PrimeValue& pv);
size_t SerializeList(std::string_view key, const PrimeValue& pv);
void SerializeRestore(std::string_view key, const PrimeValue& pk, const PrimeValue& pv,
uint64_t expire_ms);

33
src/server/journal/streamer.cc
View file

@ -224,7 +224,7 @@ void RestoreStreamer::Run() {
auto* blocking_counter = db_slice_->BlockingCounter();
std::lock_guard blocking_counter_guard(*blocking_counter);
WriteBucket(it);
stats_.buckets_loop += WriteBucket(it);
});
if (++last_yield >= 100) {
@ -232,10 +232,19 @@ void RestoreStreamer::Run() {
last_yield = 0;
}
} while (cursor);
VLOG(1) << "RestoreStreamer finished loop of " << my_slots_.ToSlotRanges().ToString()
<< ", shard " << db_slice_->shard_id() << ". Buckets looped " << stats_.buckets_loop;
}
void RestoreStreamer::SendFinalize(long attempt) {
VLOG(1) << "RestoreStreamer LSN opcode for : " << db_slice_->shard_id() << " attempt " << attempt;
VLOG(1) << "RestoreStreamer LSN of " << my_slots_.ToSlotRanges().ToString() << ", shard "
<< db_slice_->shard_id() << " attempt " << attempt << " with " << stats_.commands
<< " commands. Buckets looped " << stats_.buckets_loop << ", buckets on_db_update "
<< stats_.buckets_on_db_update << ", buckets skipped " << stats_.buckets_skipped
<< ", buckets written " << stats_.buckets_written << ". Keys skipped "
<< stats_.keys_skipped << ", keys written " << stats_.keys_written;
journal::Entry entry(journal::Op::LSN, attempt);
io::StringSink sink;
@ -285,14 +294,19 @@ bool RestoreStreamer::ShouldWrite(SlotId slot_id) const {
return my_slots_.Contains(slot_id);
}
void RestoreStreamer::WriteBucket(PrimeTable::bucket_iterator it) {
bool RestoreStreamer::WriteBucket(PrimeTable::bucket_iterator it) {
bool written = false;
if (it.GetVersion() < snapshot_version_) {
stats_.buckets_written++;
it.SetVersion(snapshot_version_);
string key_buffer; // we can reuse it
for (; !it.is_done(); ++it) {
const auto& pv = it->second;
string_view key = it->first.GetSlice(&key_buffer);
if (ShouldWrite(key)) {
stats_.keys_written++;
uint64_t expire = 0;
if (pv.HasExpire()) {
auto eit = db_slice_->databases()[0]->expire.Find(it->first);
@ -300,10 +314,17 @@ void RestoreStreamer::WriteBucket(PrimeTable::bucket_iterator it) {
}
WriteEntry(key, it->first, pv, expire);
written = true;
} else {
stats_.keys_skipped++;
}
}
} else {
stats_.buckets_skipped++;
}
ThrottleIfNeeded();
return written;
}
void RestoreStreamer::OnDbChange(DbIndex db_index, const DbSlice::ChangeReq& req) {
@ -313,12 +334,12 @@ void RestoreStreamer::OnDbChange(DbIndex db_index, const DbSlice::ChangeReq& req
PrimeTable* table = db_slice_->GetTables(0).first;
if (const PrimeTable::bucket_iterator* bit = req.update()) {
WriteBucket(*bit);
stats_.buckets_on_db_update += WriteBucket(*bit);
} else {
string_view key = get<string_view>(req.change);
table->CVCUponInsert(snapshot_version_, key, [this](PrimeTable::bucket_iterator it) {
DCHECK_LT(it.GetVersion(), snapshot_version_);
WriteBucket(it);
stats_.buckets_on_db_update += WriteBucket(it);
});
}
}
@ -331,7 +352,7 @@ void RestoreStreamer::WriteEntry(string_view key, const PrimeValue& pk, const Pr
ThrottleIfNeeded();
},
ServerState::tlocal()->serialization_max_chunk_size);
serializer.SerializeEntry(key, pk, pv, expire_ms);
stats_.commands += serializer.SerializeEntry(key, pk, pv, expire_ms);
}
} // namespace dfly

16
src/server/journal/streamer.h
View file

@ -101,11 +101,22 @@ class RestoreStreamer : public JournalStreamer {
bool ShouldWrite(std::string_view key) const;
bool ShouldWrite(SlotId slot_id) const;
// Returns whether anything was written
void WriteBucket(PrimeTable::bucket_iterator it);
// Returns true if any entry was actually written
bool WriteBucket(PrimeTable::bucket_iterator it);
void WriteEntry(std::string_view key, const PrimeValue& pk, const PrimeValue& pv,
uint64_t expire_ms);
struct Stats {
size_t buckets_skipped = 0;
size_t buckets_written = 0;
size_t buckets_loop = 0;
size_t buckets_on_db_update = 0;
size_t keys_written = 0;
size_t keys_skipped = 0;
size_t commands = 0;
};
DbSlice* db_slice_;
DbTableArray db_array_;
uint64_t snapshot_version_ = 0;
@ -113,6 +124,7 @@ class RestoreStreamer : public JournalStreamer {
bool fiber_cancelled_ = false;
bool snapshot_finished_ = false;
ThreadLocalMutex big_value_mu_;
Stats stats_;
};
} // namespace dfly