feat(server): Use hashtags for sharding in emulated cluster mode. (#1602)

This PR would have been 1 line change instead of 11 files, but it
required some plumbing and refactoring:
* Now ClusterConfig is aware of emulated cluster mode
* As a result, this API was moved from ClusterFamily
* And so was the flag & its parsing
* ClusterFamily doesn't need is_emulated_cluster_ member
* ServerFamily no longer needs ClusterFamily* member (because the API is
  static)
* I also changed `ClusterConfig::IsClusterEnabled()` to
  `ClusterConfig::IsEnabled()` to be shorter

src/server/cluster/cluster_config.cc

@@ -8,14 +8,52 @@ extern "C" {
 #include <shared_mutex>
 #include <string_view>
 
+#include "base/flags.h"
 #include "base/logging.h"
 #include "cluster_config.h"
 
 using namespace std;
 
-namespace dfly {
+ABSL_FLAG(string, cluster_mode, "", "Cluster mode supported. Default: \"\"");
 
-bool ClusterConfig::cluster_enabled = false;
+namespace dfly {
+namespace {
+enum class ClusterMode {
+  kUninitialized,
+  kNoCluster,
+  kEmulatedCluster,
+  kRealCluster,
+};
+
+ClusterMode cluster_mode = ClusterMode::kUninitialized;
+}  // namespace
+
+void ClusterConfig::Initialize() {
+  string cluster_mode_str = absl::GetFlag(FLAGS_cluster_mode);
+
+  if (cluster_mode_str == "emulated") {
+    cluster_mode = ClusterMode::kEmulatedCluster;
+  } else if (cluster_mode_str == "yes") {
+    cluster_mode = ClusterMode::kRealCluster;
+  } else if (cluster_mode_str.empty()) {
+    cluster_mode = ClusterMode::kNoCluster;
+  } else {
+    LOG(ERROR) << "Invalid value for flag --cluster_mode. Exiting...";
+    exit(1);
+  }
+}
+
+bool ClusterConfig::IsEnabled() {
+  return cluster_mode == ClusterMode::kRealCluster;
+}
+
+bool ClusterConfig::IsEmulated() {
+  return cluster_mode == ClusterMode::kEmulatedCluster;
+}
+
+bool ClusterConfig::IsEnabledOrEmulated() {
+  return IsEnabled() || IsEmulated();
+}
 
 string_view ClusterConfig::KeyTag(string_view key) {
   size_t start = key.find('{');

src/server/cluster/cluster_config.h

@@ -46,13 +46,10 @@ class ClusterConfig {
 
   static SlotId KeySlot(std::string_view key);
 
-  static void EnableCluster() {
-    cluster_enabled = true;
-  }
-
-  static bool IsClusterEnabled() {
-    return cluster_enabled;
-  }
+  static void Initialize();
+  static bool IsEnabled();
+  static bool IsEmulated();
+  static bool IsEnabledOrEmulated();
 
   // If the key contains the {...} pattern, return only the part between { and }
   static std::string_view KeyTag(std::string_view key);
@@ -82,8 +79,6 @@ class ClusterConfig {
     bool owned_by_me = false;
   };
 
-  static bool cluster_enabled;
-
   ClusterConfig() = default;
 
   ClusterShards config_;

src/server/cluster/cluster_family.cc

@@ -24,9 +24,6 @@
 #include "server/server_family.h"
 #include "server/server_state.h"
 
-ABSL_FLAG(std::string, cluster_mode, "",
-          "Cluster mode supported."
-          "default: \"\"");
 ABSL_FLAG(std::string, cluster_announce_ip, "", "ip that cluster commands announce to the client");
 
 ABSL_DECLARE_FLAG(uint32_t, port);
@@ -52,26 +49,13 @@ thread_local shared_ptr<ClusterConfig> tl_cluster_config;
 
 ClusterFamily::ClusterFamily(ServerFamily* server_family) : server_family_(server_family) {
   CHECK_NOTNULL(server_family_);
-  string cluster_mode = absl::GetFlag(FLAGS_cluster_mode);
-
-  if (cluster_mode == "emulated") {
-    is_emulated_cluster_ = true;
-  } else if (cluster_mode == "yes") {
-    ClusterConfig::EnableCluster();
-  } else if (!cluster_mode.empty()) {
-    LOG(ERROR) << "invalid cluster_mode. Exiting...";
-    exit(1);
-  }
+  ClusterConfig::Initialize();
 }
 
 ClusterConfig* ClusterFamily::cluster_config() {
   return tl_cluster_config.get();
 }
 
-bool ClusterFamily::IsEnabledOrEmulated() const {
-  return is_emulated_cluster_ || ClusterConfig::IsClusterEnabled();
-}
-
 ClusterShard ClusterFamily::GetEmulatedShardInfo(ConnectionContext* cntx) const {
   ClusterShard info{
       .slot_ranges = {{.start = 0, .end = ClusterConfig::kMaxSlotNum}},
@@ -170,7 +154,7 @@ void ClusterShardsImpl(const ClusterShards& config, ConnectionContext* cntx) {
 }  // namespace
 
 void ClusterFamily::ClusterShards(ConnectionContext* cntx) {
-  if (is_emulated_cluster_) {
+  if (ClusterConfig::IsEmulated()) {
     return ClusterShardsImpl({GetEmulatedShardInfo(cntx)}, cntx);
   } else if (tl_cluster_config != nullptr) {
     return ClusterShardsImpl(tl_cluster_config->GetConfig(), cntx);
@@ -213,7 +197,7 @@ void ClusterSlotsImpl(const ClusterShards& config, ConnectionContext* cntx) {
 }  // namespace
 
 void ClusterFamily::ClusterSlots(ConnectionContext* cntx) {
-  if (is_emulated_cluster_) {
+  if (ClusterConfig::IsEmulated()) {
     return ClusterSlotsImpl({GetEmulatedShardInfo(cntx)}, cntx);
   } else if (tl_cluster_config != nullptr) {
     return ClusterSlotsImpl(tl_cluster_config->GetConfig(), cntx);
@@ -266,7 +250,7 @@ void ClusterNodesImpl(const ClusterShards& config, string_view my_id, Connection
 }  // namespace
 
 void ClusterFamily::ClusterNodes(ConnectionContext* cntx) {
-  if (is_emulated_cluster_) {
+  if (ClusterConfig::IsEmulated()) {
     return ClusterNodesImpl({GetEmulatedShardInfo(cntx)}, server_family_->master_id(), cntx);
   } else if (tl_cluster_config != nullptr) {
     return ClusterNodesImpl(tl_cluster_config->GetConfig(), server_family_->master_id(), cntx);
@@ -328,7 +312,7 @@ void ClusterInfoImpl(const ClusterShards& config, ConnectionContext* cntx) {
 }  // namespace
 
 void ClusterFamily::ClusterInfo(ConnectionContext* cntx) {
-  if (is_emulated_cluster_) {
+  if (ClusterConfig::IsEmulated()) {
     return ClusterInfoImpl({GetEmulatedShardInfo(cntx)}, cntx);
   } else if (tl_cluster_config != nullptr) {
     return ClusterInfoImpl(tl_cluster_config->GetConfig(), cntx);
@@ -353,7 +337,7 @@ void ClusterFamily::Cluster(CmdArgList args, ConnectionContext* cntx) {
   ToUpper(&args[0]);
   string_view sub_cmd = ArgS(args, 0);
 
-  if (!is_emulated_cluster_ && !ClusterConfig::IsClusterEnabled()) {
+  if (!ClusterConfig::IsEnabledOrEmulated()) {
     return (*cntx)->SendError(kClusterDisabled);
   }
 
@@ -375,21 +359,21 @@ void ClusterFamily::Cluster(CmdArgList args, ConnectionContext* cntx) {
 }
 
 void ClusterFamily::ReadOnly(CmdArgList args, ConnectionContext* cntx) {
-  if (!is_emulated_cluster_) {
+  if (!ClusterConfig::IsEmulated()) {
     return (*cntx)->SendError(kClusterDisabled);
   }
   (*cntx)->SendOk();
 }
 
 void ClusterFamily::ReadWrite(CmdArgList args, ConnectionContext* cntx) {
-  if (!is_emulated_cluster_) {
+  if (!ClusterConfig::IsEmulated()) {
     return (*cntx)->SendError(kClusterDisabled);
   }
   (*cntx)->SendOk();
 }
 
 void ClusterFamily::DflyCluster(CmdArgList args, ConnectionContext* cntx) {
-  if (!is_emulated_cluster_ && !ClusterConfig::IsClusterEnabled()) {
+  if (!ClusterConfig::IsEnabledOrEmulated()) {
     return (*cntx)->SendError(kClusterDisabled);
   }
 

src/server/cluster/cluster_family.h

@@ -22,8 +22,6 @@ class ClusterFamily {
 
   void Register(CommandRegistry* registry);
 
-  bool IsEnabledOrEmulated() const;
-
   // Returns a thread-local pointer.
   ClusterConfig* cluster_config();
 
@@ -50,7 +48,6 @@ class ClusterFamily {
 
   ClusterConfig::ClusterShard GetEmulatedShardInfo(ConnectionContext* cntx) const;
 
-  bool is_emulated_cluster_ = false;
   ServerFamily* server_family_ = nullptr;
 };
 

src/server/db_slice.cc

@@ -55,7 +55,7 @@ void PerformDeletion(PrimeIterator del_it, ExpireIterator exp_it, EngineShard* s
   if (pv.ObjType() == OBJ_STRING)
     stats.strval_memory_usage -= value_heap_size;
 
-  if (ClusterConfig::IsClusterEnabled()) {
+  if (ClusterConfig::IsEnabled()) {
     string tmp;
     string_view key = del_it->first.GetSlice(&tmp);
     SlotId sid = ClusterConfig::KeySlot(key);
@@ -340,7 +340,7 @@ pair<PrimeIterator, ExpireIterator> DbSlice::FindExt(const Context& cntx, string
   events_.hits++;
   db.top_keys.Touch(key);
 
-  if (ClusterConfig::IsClusterEnabled()) {
+  if (ClusterConfig::IsEnabled()) {
     db.slots_stats[ClusterConfig::KeySlot(key)].total_reads += 1;
   }
 
@@ -438,7 +438,7 @@ tuple<PrimeIterator, ExpireIterator, bool> DbSlice::AddOrFind2(const Context& cn
 
     it.SetVersion(NextVersion());
     memory_budget_ = evp.mem_budget() + evicted_obj_bytes;
-    if (ClusterConfig::IsClusterEnabled()) {
+    if (ClusterConfig::IsEnabled()) {
       SlotId sid = ClusterConfig::KeySlot(key);
       db.slots_stats[sid].key_count += 1;
     }
@@ -877,7 +877,7 @@ void DbSlice::PostUpdate(DbIndex db_ind, PrimeIterator it, std::string_view key,
 
   ++events_.update;
 
-  if (ClusterConfig::IsClusterEnabled()) {
+  if (ClusterConfig::IsEnabled()) {
     db.slots_stats[ClusterConfig::KeySlot(key)].total_writes += 1;
   }
 }

src/server/engine_shard_set.h

@@ -349,7 +349,7 @@ template <typename U> void EngineShardSet::RunBlockingInParallel(U&& func) {
 }
 
 inline ShardId Shard(std::string_view v, ShardId shard_num) {
-  if (ClusterConfig::IsClusterEnabled()) {
+  if (ClusterConfig::IsEnabledOrEmulated()) {
     v = ClusterConfig::KeyTag(v);
   }
   XXH64_hash_t hash = XXH64(v.data(), v.size(), 120577240643ULL);

src/server/generic_family.cc

@@ -1165,7 +1165,7 @@ void GenericFamily::Select(CmdArgList args, ConnectionContext* cntx) {
   if (!absl::SimpleAtoi(key, &index)) {
     return (*cntx)->SendError(kInvalidDbIndErr);
   }
-  if (ClusterConfig::IsClusterEnabled() && index != 0) {
+  if (ClusterConfig::IsEnabled() && index != 0) {
     return (*cntx)->SendError("SELECT is not allowed in cluster mode");
   }
   if (index < 0 || index >= absl::GetFlag(FLAGS_dbnum)) {

src/server/main_service.cc

@@ -631,7 +631,7 @@ void Service::Init(util::AcceptServer* acceptor, std::vector<facade::Listener*>
   request_latency_usec.Init(&pp_);
   StringFamily::Init(&pp_);
   GenericFamily::Init(&pp_);
-  server_family_.Init(acceptor, std::move(listeners), &cluster_family_);
+  server_family_.Init(acceptor, std::move(listeners));
 
   ChannelStore* cs = new ChannelStore{};
   pp_.Await(
@@ -849,7 +849,7 @@ bool Service::VerifyCommand(const CommandId* cid, CmdArgList args, ConnectionCon
     }
   }
 
-  if (ClusterConfig::IsClusterEnabled() && !CheckKeysOwnership(cid, args.subspan(1), dfly_cntx)) {
+  if (ClusterConfig::IsEnabled() && !CheckKeysOwnership(cid, args.subspan(1), dfly_cntx)) {
     return false;
   }
 

src/server/server_family.cc

@@ -526,13 +526,11 @@ ServerFamily::ServerFamily(Service* service) : service_(*service) {
 ServerFamily::~ServerFamily() {
 }
 
-void ServerFamily::Init(util::AcceptServer* acceptor, std::vector<facade::Listener*> listeners,
-                        ClusterFamily* cluster_family) {
+void ServerFamily::Init(util::AcceptServer* acceptor, std::vector<facade::Listener*> listeners) {
   CHECK(acceptor_ == nullptr);
   acceptor_ = acceptor;
   listeners_ = std::move(listeners);
   dfly_cmd_ = make_unique<DflyCmd>(this);
-  cluster_family_ = cluster_family;
 
   pb_task_ = shard_set->pool()->GetNextProactor();
   if (pb_task_->GetKind() == ProactorBase::EPOLL) {
@@ -1863,7 +1861,7 @@ void ServerFamily::Info(CmdArgList args, ConnectionContext* cntx) {
 
   if (should_enter("CLUSTER")) {
     ADD_HEADER("# Cluster");
-    append("cluster_enabled", cluster_family_->IsEnabledOrEmulated());
+    append("cluster_enabled", ClusterConfig::IsEnabledOrEmulated());
   }
 
   (*cntx)->SendBulkString(info);

src/server/server_family.h

@@ -92,8 +92,7 @@ class ServerFamily {
   explicit ServerFamily(Service* service);
   ~ServerFamily();
 
-  void Init(util::AcceptServer* acceptor, std::vector<facade::Listener*> listeners,
-            ClusterFamily* cluster_family);
+  void Init(util::AcceptServer* acceptor, std::vector<facade::Listener*> listeners);
   void Register(CommandRegistry* registry);
   void Shutdown();
 
@@ -214,7 +213,6 @@ class ServerFamily {
   std::unique_ptr<ScriptMgr> script_mgr_;
   std::unique_ptr<journal::Journal> journal_;
   std::unique_ptr<DflyCmd> dfly_cmd_;
-  ClusterFamily* cluster_family_ = nullptr;  // Not owned
 
   std::string master_id_;
 

src/server/table.cc

@@ -46,7 +46,7 @@ DbTable::DbTable(PMR_NS::memory_resource* mr)
     : prime(kInitSegmentLog, detail::PrimeTablePolicy{}, mr),
       expire(0, detail::ExpireTablePolicy{}, mr), mcflag(0, detail::ExpireTablePolicy{}, mr),
       top_keys({.enabled = absl::GetFlag(FLAGS_enable_top_keys_tracking)}) {
-  if (ClusterConfig::IsClusterEnabled()) {
+  if (ClusterConfig::IsEnabled()) {
     slots_stats.resize(ClusterConfig::kMaxSlotNum + 1);
   }
 }