mirrors/dragonfly
1
0
Fork 0
mirror of https://github.com/dragonflydb/dragonfly.git synced 2025-05-11 10:25:47 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

feat(tiering): add background offload step (#2504)

Browse source 
* feat(tiering): add background offload step

Signed-off-by: adi_holden <adi@dragonflydb.io
This commit is contained in:
adiholden 2024-02-14 14:28:41 +02:00 • committed by GitHub
parent b18fe8c0a8
commit 32e8d49123
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 261 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

19
src/server/engine_shard_set.cc
View file

@ -38,6 +38,9 @@ ABSL_FLAG(dfly::MemoryBytesFlag, tiered_max_file_size, dfly::MemoryBytesFlag{},
"0 - means the program will automatically determine its maximum file size. "
"default: 0");
ABSL_FLAG(float, tiered_offload_threshold, 0.5,
"The ratio of used/max memory above which we start offloading values to disk");
ABSL_FLAG(uint32_t, hz, 100,
"Base frequency at which the server performs other background tasks. "
"Warning: not advised to decrease in production.");
@ -602,7 +605,9 @@ void EngineShard::Heartbeat() {
ttl_delete_target = kTtlDeleteLimit * double(deleted) / (double(traversed) + 10);
}
ssize_t redline = (max_memory_limit * kRedLimitFactor) / shard_set->size();
ssize_t eviction_redline = (max_memory_limit * kRedLimitFactor) / shard_set->size();
size_t tiering_redline =
(max_memory_limit * GetFlag(FLAGS_tiered_offload_threshold)) / shard_set->size();
DbContext db_cntx;
db_cntx.time_now_ms = GetCurrentTimeMs();
@ -620,8 +625,16 @@ void EngineShard::Heartbeat() {
}
// if our budget is below the limit
if (db_slice_.memory_budget() < redline) {
db_slice_.FreeMemWithEvictionStep(i, redline - db_slice_.memory_budget());
if (db_slice_.memory_budget() < eviction_redline) {
db_slice_.FreeMemWithEvictionStep(i, eviction_redline - db_slice_.memory_budget());
}
if (tiered_storage_) {
size_t offload_bytes = 0;
if (UsedMemory() > tiering_redline) {
offload_bytes = UsedMemory() - tiering_redline;
}
db_slice_.ScheduleForOffloadStep(i, offload_bytes);
}
}