mirrors/dragonfly
1
0
Fork 0
mirror of https://github.com/dragonflydb/dragonfly.git synced 2025-05-11 10:25:47 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

feat: support GCS storage for saving/loading snapshots (#4064)

Browse source
This commit is contained in:
Roman Gershman 2024-11-06 12:11:59 +02:00 • committed by GitHub
parent ae3faf59fb
commit f8b3fa0d7b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 35 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

2
helio

@ -1 +1 @@
Subproject commit 6e77555e7deb619fbd3398d085f8c604aa71869b Subproject commit 438c86139eac2e6400f2aae1d46cff03b31c166f

41
src/server/detail/snapshot_storage.cc
View file

@ -236,9 +236,7 @@ GcsSnapshotStorage::~GcsSnapshotStorage() {
} }
error_code GcsSnapshotStorage::Init(unsigned connect_ms) { error_code GcsSnapshotStorage::Init(unsigned connect_ms) {
fb2::ProactorBase* proactor = fb2::ProactorBase::me(); error_code ec = creds_provider_.Init(connect_ms);
CHECK(proactor);
error_code ec = creds_provider_.Init(connect_ms, proactor);
if (ec) if (ec)
return ec; return ec;
@ -282,10 +280,39 @@ io::ReadonlyFileOrError GcsSnapshotStorage::OpenReadFile(const std::string& path
return cloud::OpenReadGcsFile(bucket, key, opts); return cloud::OpenReadGcsFile(bucket, key, opts);
} }
io::Result<std::string, GenericError> GcsSnapshotStorage::LoadPath(std::string_view dir, io::Result<std::string, GenericError> GcsSnapshotStorage::LoadPath(string_view dir,
std::string_view dbfilename) { string_view dbfilename) {
LOG(ERROR) << "Load snapshot: Searching for snapshot in GCS path: " << dir << "/" << dbfilename; if (dbfilename.empty())
return nonstd::make_unexpected(GenericError("Not supported")); return "";
auto [bucket_name, prefix] = GetBucketPath(dir);
fb2::ProactorBase* proactor = shard_set->pool()->GetNextProactor();
io::Result<vector<SnapStat>, GenericError> keys =
proactor->Await([this, proactor, bucket_name = bucket_name,
prefix = prefix]() -> io::Result<vector<SnapStat>, GenericError> {
cloud::GCS gcs(&creds_provider_, ctx_, proactor);
vector<SnapStat> res;
error_code ec =
gcs.List(bucket_name, prefix, false, [&res](const cloud::StorageListItem& item) {
res.emplace_back(SnapStat{string(item.key), item.mtime_ns});
});
if (ec)
return nonstd::make_unexpected(GenericError(ec, "Failed to list objects"));
return res;
});
if (!keys) {
return nonstd::make_unexpected(keys.error());
}
auto match_key = FindMatchingFile(prefix, dbfilename, *keys);
if (!match_key.empty()) {
return absl::StrCat(kGCSPrefix, bucket_name, "/", match_key);
}
return nonstd::make_unexpected(GenericError(
std::make_error_code(std::errc::no_such_file_or_directory), "Snapshot not found"));
} }
io::Result<vector<string>, GenericError> GcsSnapshotStorage::ExpandFromPath( io::Result<vector<string>, GenericError> GcsSnapshotStorage::ExpandFromPath(