mirrors/dragonfly
1
0
Fork 0
mirror of https://github.com/dragonflydb/dragonfly.git synced 2025-05-11 02:15:45 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

fix(server): fix the bug where DF went over maxmemory limit (#683)

Browse source 
fix(server): fix the bug where we went over maxmemory limit significantly

Reproducible scenario: `zipfantest -c 20000000 -d 2000 -a 0.95 -p 10 -upper_bound 2000000` when
dragonfly runs with: `--maxmemory=1G --cache_mode --proactor_threads=2`

1. Inside CanGrow() check we did not account for the additional segment when doing the test against
   the memory budget.
2. Our soft budget limit was too low, preventing more detailed checks to stop the allocation.

Signed-off-by: Roman Gershman <roman@dragonflydb.io>
This commit is contained in:
Roman Gershman 2023-01-16 12:00:14 +02:00 • committed by GitHub
parent 86a4e94ec7
commit 96100382b9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 7 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

11
src/server/db_slice.cc
View file

@ -87,7 +87,7 @@ class PrimeEvictionPolicy {
// A hook function that is called every time a segment is full and requires splitting. // A hook function that is called every time a segment is full and requires splitting.
void RecordSplit(PrimeTable::Segment_t* segment) { void RecordSplit(PrimeTable::Segment_t* segment) {
mem_budget_ -= PrimeTable::kSegBytes; mem_budget_ -= PrimeTable::kSegBytes;
DVLOG(1) << "split: " << segment->SlowSize() << "/" << segment->capacity(); DVLOG(2) << "split: " << segment->SlowSize() << "/" << segment->capacity();
} }
bool CanGrow(const PrimeTable& tbl) const; bool CanGrow(const PrimeTable& tbl) const;
@ -139,8 +139,11 @@ bool PrimeEvictionPolicy::CanGrow(const PrimeTable& tbl) const {
// we estimate how much memory we will take with the current capacity // we estimate how much memory we will take with the current capacity
// even though we may currently use less memory. // even though we may currently use less memory.
// see https://github.com/dragonflydb/dragonfly/issues/256#issuecomment-1227095503 // see https://github.com/dragonflydb/dragonfly/issues/256#issuecomment-1227095503
size_t available = tbl.capacity() - tbl.size(); size_t new_available = (tbl.capacity() - tbl.size()) + PrimeTable::kSegCapacity;
return mem_budget_ > int64_t(PrimeTable::kSegBytes + db_slice_->bytes_per_object() * available); bool res = mem_budget_ >
int64_t(PrimeTable::kSegBytes + db_slice_->bytes_per_object() * new_available * 1.1);
VLOG(1) << "available: " << new_available << ", res: " << res;
return res;
} }
unsigned PrimeEvictionPolicy::GarbageCollect(const PrimeTable::HotspotBuckets& eb, PrimeTable* me) { unsigned PrimeEvictionPolicy::GarbageCollect(const PrimeTable::HotspotBuckets& eb, PrimeTable* me) {
@ -232,7 +235,7 @@ DbSlice::DbSlice(uint32_t index, bool caching_mode, EngineShard* owner)
db_arr_.emplace_back(); db_arr_.emplace_back();
CreateDb(0); CreateDb(0);
expire_base_[0] = expire_base_[1] = 0; expire_base_[0] = expire_base_[1] = 0;
soft_budget_limit_ = (0.1 * max_memory_limit / shard_set->size()); soft_budget_limit_ = (0.3 * max_memory_limit / shard_set->size());
} }
DbSlice::~DbSlice() { DbSlice::~DbSlice() {