mirror of
https://github.com/dragonflydb/dragonfly.git
synced 2025-05-11 10:25:47 +02:00
feat(server): Buffered journal serializers (#619)
This commit is contained in:
Vladislav
GitHub
parent
f204f1c670
commit
7788600c9b
13 changed files with 240 additions and 189 deletions
|
|
@ -18,154 +18,180 @@ using namespace std;
|
|||
|
||||
namespace dfly {
|
||||
|
||||
JournalWriter::JournalWriter(io::Sink* sink, std::optional<DbIndex> dbid)
|
||||
: sink_{sink}, cur_dbid_{dbid} {
|
||||
std::error_code JournalWriter::Flush(io::Sink* sink) {
|
||||
if (auto ec = sink->Write(buf_.InputBuffer()); ec)
|
||||
return ec;
|
||||
buf_.Clear();
|
||||
return {};
|
||||
}
|
||||
|
||||
error_code JournalWriter::Write(uint64_t v) {
|
||||
base::IoBuf& JournalWriter::Accumulated() {
|
||||
return buf_;
|
||||
}
|
||||
|
||||
void JournalWriter::Write(uint64_t v) {
|
||||
uint8_t buf[10];
|
||||
unsigned len = WritePackedUInt(v, buf);
|
||||
return sink_->Write(io::Bytes{buf, len});
|
||||
buf_.EnsureCapacity(sizeof(buf));
|
||||
memcpy(buf_.AppendBuffer().data(), buf, len);
|
||||
buf_.CommitWrite(len);
|
||||
}
|
||||
|
||||
error_code JournalWriter::Write(std::string_view sv) {
|
||||
RETURN_ON_ERR(Write(sv.size()));
|
||||
return sink_->Write(io::Buffer(sv));
|
||||
void JournalWriter::Write(std::string_view sv) {
|
||||
Write(sv.size());
|
||||
buf_.EnsureCapacity(sv.size());
|
||||
memcpy(buf_.AppendBuffer().data(), sv.data(), sv.size());
|
||||
buf_.CommitWrite(sv.size());
|
||||
}
|
||||
|
||||
error_code JournalWriter::Write(CmdArgList args) {
|
||||
RETURN_ON_ERR(Write(args.size()));
|
||||
void JournalWriter::Write(CmdArgList args) {
|
||||
Write(args.size());
|
||||
for (auto v : args)
|
||||
RETURN_ON_ERR(Write(facade::ToSV(v)));
|
||||
|
||||
return std::error_code{};
|
||||
Write(facade::ToSV(v));
|
||||
}
|
||||
|
||||
error_code JournalWriter::Write(std::pair<std::string_view, ArgSlice> args) {
|
||||
void JournalWriter::Write(std::pair<std::string_view, ArgSlice> args) {
|
||||
auto [cmd, tail_args] = args;
|
||||
|
||||
RETURN_ON_ERR(Write(1 + tail_args.size()));
|
||||
RETURN_ON_ERR(Write(cmd));
|
||||
Write(1 + tail_args.size());
|
||||
Write(cmd);
|
||||
for (auto v : tail_args)
|
||||
RETURN_ON_ERR(Write(v));
|
||||
|
||||
return std::error_code{};
|
||||
Write(v);
|
||||
}
|
||||
|
||||
error_code JournalWriter::Write(std::monostate) {
|
||||
return std::error_code{};
|
||||
void JournalWriter::Write(std::monostate) {
|
||||
}
|
||||
|
||||
error_code JournalWriter::Write(const journal::Entry& entry) {
|
||||
void JournalWriter::Write(const journal::Entry& entry) {
|
||||
// Check if entry has a new db index and we need to emit a SELECT entry.
|
||||
if (entry.opcode != journal::Op::SELECT && (!cur_dbid_ || entry.dbid != *cur_dbid_)) {
|
||||
RETURN_ON_ERR(Write(journal::Entry{journal::Op::SELECT, entry.dbid}));
|
||||
Write(journal::Entry{journal::Op::SELECT, entry.dbid});
|
||||
cur_dbid_ = entry.dbid;
|
||||
}
|
||||
|
||||
RETURN_ON_ERR(Write(uint8_t(entry.opcode)));
|
||||
Write(uint8_t(entry.opcode));
|
||||
|
||||
switch (entry.opcode) {
|
||||
case journal::Op::SELECT:
|
||||
return Write(entry.dbid);
|
||||
case journal::Op::COMMAND:
|
||||
RETURN_ON_ERR(Write(entry.txid));
|
||||
RETURN_ON_ERR(Write(entry.shard_cnt));
|
||||
Write(entry.txid);
|
||||
Write(entry.shard_cnt);
|
||||
return std::visit([this](const auto& payload) { return Write(payload); }, entry.payload);
|
||||
default:
|
||||
break;
|
||||
};
|
||||
return std::error_code{};
|
||||
}
|
||||
|
||||
JournalReader::JournalReader(DbIndex dbid) : buf_{}, dbid_{dbid} {
|
||||
JournalReader::JournalReader(io::Source* source, DbIndex dbid)
|
||||
: str_buf_{}, source_{source}, buf_{4096}, dbid_{dbid} {
|
||||
}
|
||||
|
||||
void JournalReader::SetDb(DbIndex dbid) {
|
||||
dbid_ = dbid;
|
||||
}
|
||||
|
||||
template <typename UT> io::Result<UT> ReadPackedUIntTyped(io::Source* source) {
|
||||
uint64_t v;
|
||||
SET_OR_UNEXPECT(ReadPackedUInt(source), v);
|
||||
if (v > std::numeric_limits<UT>::max())
|
||||
return make_unexpected(make_error_code(errc::result_out_of_range));
|
||||
return static_cast<UT>(v);
|
||||
void JournalReader::SetSource(io::Source* source) {
|
||||
CHECK_EQ(buf_.InputLen(), 0ULL);
|
||||
source_ = source;
|
||||
}
|
||||
|
||||
io::Result<uint8_t> JournalReader::ReadU8(io::Source* source) {
|
||||
return ReadPackedUIntTyped<uint8_t>(source);
|
||||
}
|
||||
std::error_code JournalReader::EnsureRead(size_t num) {
|
||||
// Check if we already have enough.
|
||||
if (buf_.InputLen() >= num)
|
||||
return {};
|
||||
|
||||
io::Result<uint16_t> JournalReader::ReadU16(io::Source* source) {
|
||||
return ReadPackedUIntTyped<uint16_t>(source);
|
||||
}
|
||||
uint64_t remainder = num - buf_.InputLen();
|
||||
buf_.EnsureCapacity(remainder);
|
||||
|
||||
io::Result<uint32_t> JournalReader::ReadU32(io::Source* source) {
|
||||
return ReadPackedUIntTyped<uint32_t>(source);
|
||||
}
|
||||
|
||||
io::Result<uint64_t> JournalReader::ReadU64(io::Source* source) {
|
||||
return ReadPackedUIntTyped<uint64_t>(source);
|
||||
}
|
||||
|
||||
io::Result<size_t> JournalReader::ReadString(io::Source* source) {
|
||||
size_t size = 0;
|
||||
SET_OR_UNEXPECT(ReadU64(source), size);
|
||||
|
||||
buf_.EnsureCapacity(size);
|
||||
auto dest = buf_.AppendBuffer().first(size);
|
||||
uint64_t read = 0;
|
||||
SET_OR_UNEXPECT(source->Read(dest), read);
|
||||
// Try reading at least how much we need, but possibly more
|
||||
uint64_t read;
|
||||
SET_OR_RETURN(source_->ReadAtLeast(buf_.AppendBuffer(), remainder), read);
|
||||
CHECK(read >= remainder);
|
||||
|
||||
buf_.CommitWrite(read);
|
||||
if (read != size)
|
||||
return make_unexpected(std::make_error_code(std::errc::message_size));
|
||||
return {};
|
||||
}
|
||||
|
||||
template <typename UT> io::Result<UT> JournalReader::ReadUInt() {
|
||||
// Determine type and number of following bytes.
|
||||
if (auto ec = EnsureRead(1); ec)
|
||||
return make_unexpected(ec);
|
||||
PackedUIntMeta meta{buf_.InputBuffer()[0]};
|
||||
buf_.ConsumeInput(1);
|
||||
|
||||
if (auto ec = EnsureRead(meta.ByteSize()); ec)
|
||||
return make_unexpected(ec);
|
||||
|
||||
// Read and check intenger.
|
||||
uint64_t res;
|
||||
SET_OR_UNEXPECT(ReadPackedUInt(meta, buf_.InputBuffer()), res);
|
||||
buf_.ConsumeInput(meta.ByteSize());
|
||||
|
||||
if (res > std::numeric_limits<UT>::max())
|
||||
return make_unexpected(make_error_code(errc::result_out_of_range));
|
||||
return static_cast<UT>(res);
|
||||
}
|
||||
|
||||
template io::Result<uint8_t> JournalReader::ReadUInt<uint8_t>();
|
||||
template io::Result<uint16_t> JournalReader::ReadUInt<uint16_t>();
|
||||
template io::Result<uint32_t> JournalReader::ReadUInt<uint32_t>();
|
||||
template io::Result<uint64_t> JournalReader::ReadUInt<uint64_t>();
|
||||
|
||||
io::Result<size_t> JournalReader::ReadString() {
|
||||
size_t size = 0;
|
||||
SET_OR_UNEXPECT(ReadUInt<uint64_t>(), size);
|
||||
|
||||
if (auto ec = EnsureRead(size); ec)
|
||||
return make_unexpected(ec);
|
||||
|
||||
unsigned offset = str_buf_.size();
|
||||
str_buf_.resize(offset + size);
|
||||
buf_.ReadAndConsume(size, str_buf_.data() + offset);
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
std::error_code JournalReader::Read(io::Source* source, CmdArgVec* vec) {
|
||||
buf_.ConsumeInput(buf_.InputBuffer().size());
|
||||
std::error_code JournalReader::Read(CmdArgVec* vec) {
|
||||
size_t num_strings = 0;
|
||||
SET_OR_RETURN(ReadUInt<uint64_t>(), num_strings);
|
||||
vec->resize(num_strings);
|
||||
|
||||
size_t size = 0;
|
||||
SET_OR_RETURN(ReadU64(source), size);
|
||||
|
||||
vec->resize(size);
|
||||
// Read all strings consecutively.
|
||||
str_buf_.clear();
|
||||
for (auto& span : *vec) {
|
||||
size_t len;
|
||||
SET_OR_RETURN(ReadString(source), len);
|
||||
span = MutableSlice{nullptr, len};
|
||||
size_t size;
|
||||
SET_OR_RETURN(ReadString(), size);
|
||||
span = MutableSlice{nullptr, size};
|
||||
}
|
||||
|
||||
size_t offset = 0;
|
||||
// Set span pointers, now that string buffer won't reallocate.
|
||||
char* ptr = str_buf_.data();
|
||||
for (auto& span : *vec) {
|
||||
size_t len = span.size();
|
||||
auto ptr = buf_.InputBuffer().subspan(offset).data();
|
||||
span = MutableSlice{reinterpret_cast<char*>(ptr), len};
|
||||
offset += len;
|
||||
span = {ptr, span.size()};
|
||||
ptr += span.size();
|
||||
}
|
||||
|
||||
return std::error_code{};
|
||||
}
|
||||
|
||||
io::Result<journal::ParsedEntry> JournalReader::ReadEntry(io::Source* source) {
|
||||
io::Result<journal::ParsedEntry> JournalReader::ReadEntry() {
|
||||
uint8_t opcode;
|
||||
SET_OR_UNEXPECT(ReadU8(source), opcode);
|
||||
SET_OR_UNEXPECT(ReadUInt<uint8_t>(), opcode);
|
||||
|
||||
journal::ParsedEntry entry{static_cast<journal::Op>(opcode), dbid_};
|
||||
|
||||
switch (entry.opcode) {
|
||||
case journal::Op::COMMAND:
|
||||
SET_OR_UNEXPECT(ReadU64(source), entry.txid);
|
||||
SET_OR_UNEXPECT(ReadU32(source), entry.shard_cnt);
|
||||
SET_OR_UNEXPECT(ReadUInt<uint64_t>(), entry.txid);
|
||||
SET_OR_UNEXPECT(ReadUInt<uint32_t>(), entry.shard_cnt);
|
||||
entry.payload = CmdArgVec{};
|
||||
if (auto ec = Read(source, &*entry.payload); ec)
|
||||
if (auto ec = Read(&*entry.payload); ec)
|
||||
return make_unexpected(ec);
|
||||
break;
|
||||
case journal::Op::SELECT:
|
||||
SET_OR_UNEXPECT(ReadU16(source), dbid_);
|
||||
return ReadEntry(source);
|
||||
SET_OR_UNEXPECT(ReadUInt<uint16_t>(), dbid_);
|
||||
return ReadEntry();
|
||||
default:
|
||||
break;
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue