Improve eviction policy design for dash table

src/server/common.cc

@@ -52,4 +52,52 @@ const char* GlobalState::Name(S s) {
   ABSL_INTERNAL_UNREACHABLE;
 }
 
+bool ParseHumanReadableBytes(std::string_view str, int64_t* num_bytes) {
+  if (str.empty())
+    return false;
+
+  const char* cstr = str.data();
+  bool neg = (*cstr == '-');
+  if (neg) {
+    cstr++;
+  }
+  char* end;
+  double d = strtod(cstr, &end);
+
+  // If this didn't consume the entire string, fail.
+  if (end + 1 < str.end())
+    return false;
+
+  int64 scale = 1;
+  switch (*end) {
+    // NB: an int64 can only go up to <8 EB.
+    case 'E':
+      scale <<= 10;  // Fall through...
+    case 'P':
+      scale <<= 10;
+    case 'T':
+      scale <<= 10;
+    case 'G':
+      scale <<= 10;
+    case 'M':
+      scale <<= 10;
+    case 'K':
+    case 'k':
+      scale <<= 10;
+    case 'B':
+    case '\0':
+      break;  // To here.
+    default:
+      return false;
+  }
+  d *= scale;
+  if (d > kint64max || d < 0)
+    return false;
+  *num_bytes = static_cast<int64>(d + 0.5);
+  if (neg) {
+    *num_bytes = -*num_bytes;
+  }
+  return true;
+}
+
 }  // namespace dfly

src/server/common_types.h

@@ -68,4 +68,6 @@ inline void ToLower(const MutableSlice* val) {
   }
 }
 
+bool ParseHumanReadableBytes(std::string_view str, int64_t* num_bytes);
+
 }  // namespace dfly

src/server/db_slice.h

@@ -96,11 +96,14 @@ class DbSlice {
     expire_base_[generation & 1] = now;
   }
 
+  void SetMaxMemory(size_t max_memory) {
+    max_memory_ = max_memory;
+  }
+
   uint64_t expire_base() const {
     return expire_base_[0];
   }
 
-
   // returns wall clock in millis as it has been set via UpdateExpireClock.
   uint64_t Now() const {
     return now_ms_;
@@ -224,6 +227,7 @@ class DbSlice {
   uint64_t expire_base_[2];    // Used for expire logic, represents a real clock.
 
   uint64_t version_ = 1;        // Used to version entries in the PrimeTable.
+  uint64_t max_memory_ = -1;
   mutable SliceEvents events_;  // we may change this even for const operations.
 
   using LockTable = absl::flat_hash_map<std::string, IntentLock>;

src/server/dfly_main.cc

@@ -5,6 +5,8 @@
 #include <mimalloc.h>
 
 #include "base/init.h"
+#include "base/proc_util.h"
+
 #include "facade/dragonfly_listener.h"
 #include "server/main_service.h"
 #include "util/accept_server.h"
@@ -13,6 +15,8 @@
 
 DECLARE_uint32(port);
 DECLARE_uint32(memcache_port);
+DECLARE_uint64(maxmemory);
+DEFINE_bool(use_large_pages, false, "If true - uses large memory pages for allocations");
 
 using namespace util;
 using namespace std;
@@ -20,7 +24,13 @@ using namespace facade;
 
 namespace dfly {
 
-void RunEngine(ProactorPool* pool, AcceptServer* acceptor) {
+bool RunEngine(ProactorPool* pool, AcceptServer* acceptor) {
+
+  if (FLAGS_maxmemory > 0 && FLAGS_maxmemory < pool->size() * 256_MB ) {
+    LOG(ERROR) << "Max memory is less than 256MB per thread. Exiting...";
+    return false;
+  }
+
   Service service(pool);
 
   service.Init(acceptor);
@@ -34,6 +44,7 @@ void RunEngine(ProactorPool* pool, AcceptServer* acceptor) {
   acceptor->Wait();
 
   service.Shutdown();
+  return true;
 }
 
 }  // namespace dfly
@@ -48,7 +59,17 @@ int main(int argc, char* argv[]) {
 
   CHECK_GT(FLAGS_port, 0u);
 
-  mi_option_enable(mi_option_large_os_pages);
+  base::sys::KernelVersion kver;
+  base::sys::GetKernelVersion(&kver);
+
+  if (kver.major < 5 || (kver.major == 5 && kver.minor < 11)) {
+    LOG(ERROR) << "Kernel 5.11 or later is supported. Exiting...";
+    return 1;
+  }
+
+  if (FLAGS_use_large_pages) {
+    mi_option_enable(mi_option_large_os_pages);
+  }
   mi_option_enable(mi_option_show_errors);
   mi_option_set(mi_option_max_warnings, 0);
   _mi_options_init();
@@ -58,9 +79,9 @@ int main(int argc, char* argv[]) {
 
   AcceptServer acceptor(&pp);
 
-  dfly::RunEngine(&pp, &acceptor);
+  int res = dfly::RunEngine(&pp, &acceptor) ? 0 : -1;
 
   pp.Stop();
 
-  return 0;
+  return res;
 }

src/server/main_service.cc

@@ -33,9 +33,14 @@ extern "C" {
 #include "util/uring/uring_fiber_algo.h"
 #include "util/varz.h"
 
+// TODO: to move to absl flags and keep legacy flags only for glog library.
+// absl flags allow parsing of custom types and allow specifying which flags appear
+// for helpshort.
 DEFINE_uint32(port, 6380, "Redis port");
 DEFINE_uint32(memcache_port, 0, "Memcached port");
 DECLARE_string(requirepass);
+DEFINE_uint64(maxmemory, 0, "Limit on maximum-memory that is used by the database");
+
 
 namespace dfly {
 

src/server/server_family.cc

@@ -72,10 +72,11 @@ error_code CreateDirs(fs::path dir_path) {
   return ec;
 }
 
-atomic_uint64_t used_mem_peak(0);
-
 }  // namespace
 
+atomic_uint64_t used_mem_peak(0);
+atomic_uint64_t used_mem_current(0);
+
 ServerFamily::ServerFamily(Service* service) : service_(*service), ess_(service->shard_set()) {
   start_time_ = time(NULL);
   last_save_.store(start_time_, memory_order_release);
@@ -96,6 +97,8 @@ void ServerFamily::Init(util::AcceptServer* acceptor) {
     for (const auto& s : stats)
       sum += s.used_memory.load(memory_order_relaxed);
 
+    used_mem_current.store(sum, memory_order_relaxed);
+
     // Single writer, so no races.
     if (sum > used_mem_peak.load(memory_order_relaxed))
       used_mem_peak.store(sum, memory_order_relaxed);