chore: Introduce basic deadlock detector for tests (#1896)

Also, fix IOUring dependency issue in tests on non-linux platforms.

Signed-off-by: Roman Gershman <roman@dragonflydb.io>

src/server/test_utils.cc

@@ -164,6 +164,15 @@ void BaseFamilyTest::SetUp() {
   ResetService();
 }
 
+void BaseFamilyTest::TearDown() {
+  CHECK_EQ(NumLocked(), 0U);
+
+  ShutdownService();
+
+  const TestInfo* const test_info = UnitTest::GetInstance()->current_test_info();
+  LOG(INFO) << "Finishing " << test_info->name();
+}
+
 // Test hook defined in common.cc.
 void TEST_InvalidateLockHashTag();
 
@@ -171,20 +180,19 @@ void BaseFamilyTest::ResetService() {
   if (service_ != nullptr) {
     TEST_InvalidateLockHashTag();
 
-    service_->Shutdown();
+    ShutdownService();
-    service_ = nullptr;
-
-    delete shard_set;
-    shard_set = nullptr;
-
-    pp_->Stop();
   }
 
+#ifdef __linux__
   if (absl::GetFlag(FLAGS_force_epoll)) {
     pp_.reset(fb2::Pool::Epoll(num_threads_));
   } else {
     pp_.reset(fb2::Pool::IOUring(16, num_threads_));
   }
+#else
+  pp_.reset(fb2::Pool::Epoll(num_threads_));
+#endif
+
   pp_->Run();
   service_ = std::make_unique<Service>(pp_.get());
 
@@ -199,6 +207,28 @@ void BaseFamilyTest::ResetService() {
 
   const TestInfo* const test_info = UnitTest::GetInstance()->current_test_info();
   LOG(INFO) << "Starting " << test_info->name();
+
+  watchdog_fiber_ = pp_->GetNextProactor()->LaunchFiber([this] {
+    if (!watchdog_done_.WaitFor(120s)) {
+      LOG(ERROR) << "Deadlock detected!!!!";
+      absl::SetFlag(&FLAGS_alsologtostderr, true);
+      fb2::detail::FiberInterface::PrintAllFiberStackTraces();
+    }
+  });
+}
+
+void BaseFamilyTest::ShutdownService() {
+  DCHECK(service_);
+
+  service_->Shutdown();
+  service_.reset();
+  delete shard_set;
+  shard_set = nullptr;
+
+  watchdog_done_.Notify();
+  watchdog_fiber_.Join();
+
+  pp_->Stop();
 }
 
 unsigned BaseFamilyTest::NumLocked() {
@@ -214,17 +244,6 @@ unsigned BaseFamilyTest::NumLocked() {
   return count;
 }
 
-void BaseFamilyTest::TearDown() {
-  CHECK_EQ(NumLocked(), 0U);
-
-  service_->Shutdown();
-  service_.reset();
-  pp_->Stop();
-
-  const TestInfo* const test_info = UnitTest::GetInstance()->current_test_info();
-  LOG(INFO) << "Finishing " << test_info->name();
-}
-
 void BaseFamilyTest::WaitUntilLocked(DbIndex db_index, string_view key, double timeout) {
   auto step = 50us;
   auto timeout_micro = chrono::duration_cast<chrono::microseconds>(1000ms * timeout);

src/server/test_utils.h

@@ -91,6 +91,8 @@ class BaseFamilyTest : public ::testing::Test {
 
   void ResetService();
 
+  void ShutdownService();
+
   bool IsLocked(DbIndex db_index, std::string_view key) const;
   ConnectionContext::DebugInfo GetDebugInfo(const std::string& id) const;
 
@@ -139,6 +141,8 @@ class BaseFamilyTest : public ::testing::Test {
 
   std::vector<RespVec*> resp_vec_;
   bool single_response_ = true;
+  util::fb2::Fiber watchdog_fiber_;
+  util::fb2::Done watchdog_done_;
 };
 
 std::ostream& operator<<(std::ostream& os, const DbStats& stats);