mirrors/dragonfly
1
0
Fork 0
mirror of https://github.com/dragonflydb/dragonfly.git synced 2025-05-11 18:35:46 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

chore: tiering - make Modify work with cool storage (#3395)

Browse source 
1. Fully support tiered_experimental_cooling for all operations
2. Offset cool storage usage when computing memory pressure situations in Hearbeat.
3. Introduce realtime entry counting per db_slice and provide DCHECK to verify it vs the old approach.
   Later we will switch to realtime entry and free memory computations when computing bytes per object,
   and remove the old approach in CacheStats().
4. Show hit rate during the run of dfly_bench loadtest.

Signed-off-by: Roman Gershman <roman@dragonflydb.io>
This commit is contained in:
Roman Gershman 2024-07-27 14:31:29 +03:00 • committed by GitHub
parent 9d16bd6f6e
commit 6b67f44e29
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 95 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

17
src/server/engine_shard_set.cc
View file

@ -620,10 +620,15 @@ void EngineShard::Heartbeat() {
}
ssize_t eviction_redline = size_t(max_memory_limit * kRedLimitFactor) / shard_set->size();
// Offset CoolMemoryUsage when consider background offloading.
// TODO: Another approach could be is to align the approach similarly to how we do with
// FreeMemWithEvictionStep, i.e. if memory_budget is below the limit.
size_t tiering_offload_threshold =
tiered_storage_
? size_t(max_memory_limit * GetFlag(FLAGS_tiered_offload_threshold)) / shard_set->size()
: std::numeric_limits<size_t>::max();
tiered_storage_ ? tiered_storage_->CoolMemoryUsage() +
size_t(max_memory_limit * GetFlag(FLAGS_tiered_offload_threshold)) /
shard_set->size()
: std::numeric_limits<size_t>::max();
DbContext db_cntx;
db_cntx.time_now_ms = GetCurrentTimeMs();
@ -738,6 +743,7 @@ void EngineShard::CacheStats() {
}
}
DCHECK_EQ(table_memory, db_slice.table_memory());
DCHECK_EQ(entries, db_slice.entries_count());
if (tiered_storage_) {
table_memory += tiered_storage_->CoolMemoryUsage();
}
@ -764,7 +770,10 @@ bool EngineShard::ShouldThrottleForTiering() const { // see header for formula
size_t tiering_redline =
(max_memory_limit * GetFlag(FLAGS_tiered_offload_threshold)) / shard_set->size();
return UsedMemory() > tiering_redline && tiered_storage_->WriteDepthUsage() > 0.3;
// UsedMemory includes CoolMemoryUsage, so we are offsetting it to remove the cool cache impact.
return tiered_storage_->WriteDepthUsage() > 0.3 &&
(UsedMemory() > tiering_redline + tiered_storage_->CoolMemoryUsage());
}
auto EngineShard::AnalyzeTxQueue() const -> TxQueueInfo {