From bff9cf69235b59c09045f80477b60ea714d26181 Mon Sep 17 00:00:00 2001 From: Roman Gershman Date: Sun, 2 Feb 2025 21:25:01 +0200 Subject: [PATCH] chore: move QList::Node definition into dragonfly codebase (#4547) Signed-off-by: Roman Gershman --- src/core/qlist.cc | 98 +++++++++++++++++++-------------------- src/core/qlist.h | 103 ++++++++++++++++++++++++++++++++--------- src/core/qlist_test.cc | 2 +- src/server/rdb_save.cc | 2 +- 4 files changed, 130 insertions(+), 75 deletions(-) diff --git a/src/core/qlist.cc b/src/core/qlist.cc index 240a9c405..7f94151aa 100644 --- a/src/core/qlist.cc +++ b/src/core/qlist.cc @@ -98,7 +98,7 @@ bool IsLargeElement(size_t sz, int fill) { return sz > NodeNegFillLimit(fill); } -bool NodeAllowInsert(const quicklistNode* node, const int fill, const size_t sz) { +bool NodeAllowInsert(const QList::Node* node, const int fill, const size_t sz) { if (ABSL_PREDICT_FALSE(!node)) return false; @@ -114,7 +114,7 @@ bool NodeAllowInsert(const quicklistNode* node, const int fill, const size_t sz) return !quicklistNodeExceedsLimit(fill, new_sz, node->count + 1); } -bool NodeAllowMerge(const quicklistNode* a, const quicklistNode* b, const int fill) { +bool NodeAllowMerge(const QList::Node* a, const QList::Node* b, const int fill) { if (!a || !b) return false; @@ -130,8 +130,8 @@ bool NodeAllowMerge(const quicklistNode* a, const quicklistNode* b, const int fi } // the owner over entry is passed to the node. -quicklistNode* CreateRAW(int container, uint8_t* entry, size_t sz) { - quicklistNode* node = (quicklistNode*)zmalloc(sizeof(*node)); +QList::Node* CreateRAW(int container, uint8_t* entry, size_t sz) { + QList::Node* node = (QList::Node*)zmalloc(sizeof(*node)); node->entry = entry; node->count = 1; node->sz = sz; @@ -156,7 +156,7 @@ uint8_t* LP_Prepend(uint8_t* lp, string_view elem) { return lpPrepend(lp, uint_ptr(elem), elem.size()); } -quicklistNode* CreateFromSV(int container, string_view value) { +QList::Node* CreateFromSV(int container, string_view value) { uint8_t* entry = nullptr; size_t sz = 0; if (container == QUICKLIST_NODE_CONTAINER_PLAIN) { @@ -173,7 +173,7 @@ quicklistNode* CreateFromSV(int container, string_view value) { } // Returns the relative increase in size. -inline ssize_t NodeSetEntry(quicklistNode* node, uint8_t* entry) { +inline ssize_t NodeSetEntry(QList::Node* node, uint8_t* entry) { node->entry = entry; size_t new_sz = lpBytes(node->entry); ssize_t diff = new_sz - node->sz; @@ -184,7 +184,7 @@ inline ssize_t NodeSetEntry(quicklistNode* node, uint8_t* entry) { /* Compress the listpack in 'node' and update encoding details. * Returns true if listpack compressed successfully. * Returns false if compression failed or if listpack too small to compress. */ -bool CompressNode(quicklistNode* node) { +bool CompressNode(QList::Node* node) { DCHECK(node->encoding == QUICKLIST_NODE_ENCODING_RAW); DCHECK(!node->dont_compress); @@ -217,7 +217,7 @@ bool CompressNode(quicklistNode* node) { return true; } -ssize_t CompressNodeIfNeeded(quicklistNode* node) { +ssize_t CompressNodeIfNeeded(QList::Node* node) { DCHECK(node); if (node->encoding == QUICKLIST_NODE_ENCODING_RAW && !node->dont_compress) { if (CompressNode(node)) @@ -228,7 +228,7 @@ ssize_t CompressNodeIfNeeded(quicklistNode* node) { /* Uncompress the listpack in 'node' and update encoding details. * Returns 1 on successful decode, 0 on failure to decode. */ -bool DecompressNode(bool recompress, quicklistNode* node) { +bool DecompressNode(bool recompress, QList::Node* node) { node->recompress = int(recompress); void* decompressed = zmalloc(node->sz); @@ -248,7 +248,7 @@ bool DecompressNode(bool recompress, quicklistNode* node) { recompress: if true, the node will be marked for recompression after decompression. returns by how much the size of the node has increased. */ -ssize_t DecompressNodeIfNeeded(bool recompress, quicklistNode* node) { +ssize_t DecompressNodeIfNeeded(bool recompress, QList::Node* node) { if ((node) && (node)->encoding == QUICKLIST_NODE_ENCODING_LZF) { size_t compressed_sz = ((quicklistLZF*)node->entry)->sz; if (DecompressNode(recompress, node)) { @@ -258,7 +258,7 @@ ssize_t DecompressNodeIfNeeded(bool recompress, quicklistNode* node) { return 0; } -ssize_t RecompressOnly(quicklistNode* node) { +ssize_t RecompressOnly(QList::Node* node) { if (node->recompress && !node->dont_compress) { if (CompressNode(node)) return ((quicklistLZF*)node->entry)->sz - node->sz; @@ -266,7 +266,7 @@ ssize_t RecompressOnly(quicklistNode* node) { return 0; } -quicklistNode* SplitNode(quicklistNode* node, int offset, bool after, ssize_t* diff) { +QList::Node* SplitNode(QList::Node* node, int offset, bool after, ssize_t* diff) { DCHECK(node->container == QUICKLIST_NODE_CONTAINER_PACKED); size_t zl_sz = node->sz; uint8_t* entry = (uint8_t*)zmalloc(zl_sz); @@ -287,7 +287,7 @@ quicklistNode* SplitNode(quicklistNode* node, int offset, bool after, ssize_t* d node->count = lpLength(node->entry); entry = lpDeleteRange(entry, new_start, new_extent); - quicklistNode* new_node = CreateRAW(QUICKLIST_NODE_CONTAINER_PACKED, entry, lpBytes(entry)); + QList::Node* new_node = CreateRAW(QUICKLIST_NODE_CONTAINER_PACKED, entry, lpBytes(entry)); new_node->count = lpLength(new_node->entry); *diff = diff_existing; @@ -300,9 +300,6 @@ void QList::SetPackedThreshold(unsigned threshold) { packed_threshold = threshold; } -QList::QList() : fill_(-2), compress_(0), bookmark_count_(0) { -} - QList::QList(int fill, int compress) : fill_(fill), compress_(compress), bookmark_count_(0) { } @@ -338,10 +335,10 @@ QList& QList::operator=(QList&& other) { } void QList::Clear() { - quicklistNode* current = head_; + Node* current = head_; while (len_) { - quicklistNode* next = current->next; + Node* next = current->next; zfree(current->entry); zfree(current); @@ -363,7 +360,7 @@ void QList::Push(string_view value, Where where) { DCHECK(head_->prev->encoding != QUICKLIST_NODE_ENCODING_LZF); } - quicklistNode* orig = head_; + Node* orig = head_; if (where == TAIL && orig) { orig = orig->prev; } @@ -389,14 +386,14 @@ void QList::Push(string_view value, Where where) { return; } - quicklistNode* node = CreateFromSV(QUICKLIST_NODE_CONTAINER_PACKED, value); + Node* node = CreateFromSV(QUICKLIST_NODE_CONTAINER_PACKED, value); InsertNode(orig, node, opt); DCHECK(head_->prev->next == nullptr); } string QList::Pop(Where where) { DCHECK_GT(count_, 0u); - quicklistNode* node = head_; + Node* node = head_; if (where == TAIL) { node = head_->prev; } @@ -428,7 +425,7 @@ string QList::Pop(Where where) { } void QList::AppendListpack(unsigned char* zl) { - quicklistNode* node = CreateRAW(QUICKLIST_NODE_CONTAINER_PACKED, zl, lpBytes(zl)); + Node* node = CreateRAW(QUICKLIST_NODE_CONTAINER_PACKED, zl, lpBytes(zl)); node->count = lpLength(node->entry); InsertNode(_Tail(), node, AFTER); @@ -436,7 +433,7 @@ void QList::AppendListpack(unsigned char* zl) { } void QList::AppendPlain(unsigned char* data, size_t sz) { - quicklistNode* node = CreateRAW(QUICKLIST_NODE_CONTAINER_PLAIN, data, sz); + Node* node = CreateRAW(QUICKLIST_NODE_CONTAINER_PLAIN, data, sz); InsertNode(_Tail(), node, AFTER); ++count_; } @@ -464,9 +461,9 @@ bool QList::Replace(long index, std::string_view elem) { } size_t QList::MallocUsed(bool slow) const { - size_t node_size = len_ * sizeof(quicklistNode) + znallocx(sizeof(quicklist)); + size_t node_size = len_ * sizeof(Node) + znallocx(sizeof(quicklist)); if (slow) { - for (quicklistNode* node = head_; node; node = node->next) { + for (Node* node = head_; node; node = node->next) { node_size += zmalloc_usable_size(node->entry); } return node_size; @@ -490,15 +487,14 @@ void QList::Iterate(IterateFunc cb, long start, long end) const { } } -quicklistNode* QList::InsertPlainNode(quicklistNode* old_node, string_view value, - InsertOpt insert_opt) { - quicklistNode* new_node = CreateFromSV(QUICKLIST_NODE_CONTAINER_PLAIN, value); +auto QList::InsertPlainNode(Node* old_node, string_view value, InsertOpt insert_opt) -> Node* { + Node* new_node = CreateFromSV(QUICKLIST_NODE_CONTAINER_PLAIN, value); InsertNode(old_node, new_node, insert_opt); count_++; return new_node; } -void QList::InsertNode(quicklistNode* old_node, quicklistNode* new_node, InsertOpt insert_opt) { +void QList::InsertNode(Node* old_node, Node* new_node, InsertOpt insert_opt) { if (insert_opt == AFTER) { new_node->prev = old_node; if (old_node) { @@ -544,7 +540,7 @@ void QList::Insert(Iterator it, std::string_view elem, InsertOpt insert_opt) { DCHECK(it.zi_); int full = 0, at_tail = 0, at_head = 0, avail_next = 0, avail_prev = 0; - quicklistNode* node = it.current_; + Node* node = it.current_; size_t sz = elem.size(); bool after = insert_opt == AFTER; @@ -574,8 +570,8 @@ void QList::Insert(Iterator it, std::string_view elem, InsertOpt insert_opt) { } else { malloc_size_ += DecompressNodeIfNeeded(true, node); ssize_t diff_existing = 0; - quicklistNode* new_node = SplitNode(node, it.offset_, after, &diff_existing); - quicklistNode* entry_node = InsertPlainNode(node, elem, insert_opt); + Node* new_node = SplitNode(node, it.offset_, after, &diff_existing); + Node* entry_node = InsertPlainNode(node, elem, insert_opt); InsertNode(entry_node, new_node, insert_opt); malloc_size_ += diff_existing; } @@ -633,7 +629,7 @@ void QList::Insert(Iterator it, std::string_view elem, InsertOpt insert_opt) { } void QList::Replace(Iterator it, std::string_view elem) { - quicklistNode* node = it.current_; + Node* node = it.current_; unsigned char* newentry; size_t sz = elem.size(); @@ -654,7 +650,7 @@ void QList::Replace(Iterator it, std::string_view elem) { DelNode(node); } } else { /* The node is full or data is a large element */ - quicklistNode *split_node = NULL, *new_node; + Node *split_node = NULL, *new_node; node->dont_compress = 1; /* Prevent compression in InsertNode() */ /* If the entry is not at the tail, split the node at the entry's offset. */ @@ -698,7 +694,7 @@ void QList::Replace(Iterator it, std::string_view elem) { * The only way to guarantee interior nodes get compressed is to iterate * to our "interior" compress depth then compress the next node we find. * If compress depth is larger than the entire list, we return immediately. */ -void QList::Compress(quicklistNode* node) { +void QList::Compress(Node* node) { if (len_ == 0) return; @@ -713,8 +709,8 @@ void QList::Compress(quicklistNode* node) { /* Iterate until we reach compress depth for both sides of the list.a * Note: because we do length checks at the *top* of this function, * we can skip explicit null checks below. Everything exists. */ - quicklistNode* forward = head_; - quicklistNode* reverse = head_->prev; + Node* forward = head_; + Node* reverse = head_->prev; int depth = 0; int in_depth = 0; while (depth++ < compress_) { @@ -751,9 +747,9 @@ void QList::Compress(quicklistNode* node) { * * Returns the new 'center' after merging. */ -quicklistNode* QList::MergeNodes(quicklistNode* center) { - quicklistNode *prev = NULL, *prev_prev = NULL, *next = NULL; - quicklistNode *next_next = NULL, *target = NULL; +auto QList::MergeNodes(Node* center) -> Node* { + Node *prev = NULL, *prev_prev = NULL, *next = NULL; + Node *next_next = NULL, *target = NULL; if (center->prev) { prev = center->prev; @@ -808,12 +804,12 @@ quicklistNode* QList::MergeNodes(quicklistNode* center) { * * Returns the input node picked to merge against or NULL if * merging was not possible. */ -quicklistNode* QList::ListpackMerge(quicklistNode* a, quicklistNode* b) { +auto QList::ListpackMerge(Node* a, Node* b) -> Node* { malloc_size_ += DecompressNodeIfNeeded(false, a); malloc_size_ += DecompressNodeIfNeeded(false, b); if ((lpMerge(&a->entry, &b->entry))) { - /* We merged listpacks! Now remove the unused quicklistNode. */ - quicklistNode *keep = NULL, *nokeep = NULL; + /* We merged listpacks! Now remove the unused Node. */ + Node *keep = NULL, *nokeep = NULL; if (!a->entry) { nokeep = a; keep = b; @@ -837,7 +833,7 @@ quicklistNode* QList::ListpackMerge(quicklistNode* a, quicklistNode* b) { return NULL; } -void QList::DelNode(quicklistNode* node) { +void QList::DelNode(Node* node) { if (node->next) node->next->prev = node->prev; @@ -872,7 +868,7 @@ void QList::DelNode(quicklistNode* node) { * * Returns true if the entire node was deleted, false if node still exists. * Also updates in/out param 'p' with the next offset in the listpack. */ -bool QList::DelPackedIndex(quicklistNode* node, uint8_t* p) { +bool QList::DelPackedIndex(Node* node, uint8_t* p) { DCHECK(!QL_NODE_IS_PLAIN(node)); if (node->count == 1) { @@ -905,7 +901,7 @@ auto QList::GetIterator(Where where) const -> Iterator { } auto QList::GetIterator(long idx) const -> Iterator { - quicklistNode* n; + Node* n; unsigned long long accum = 0; int forward = idx < 0 ? 0 : 1; /* < 0 -> reverse, 0+ -> forward */ uint64_t index = forward ? idx : (-idx) - 1; @@ -959,9 +955,9 @@ auto QList::GetIterator(long idx) const -> Iterator { auto QList::Erase(Iterator it) -> Iterator { DCHECK(it.current_); - quicklistNode* node = it.current_; - quicklistNode* prev = node->prev; - quicklistNode* next = node->next; + Node* node = it.current_; + Node* prev = node->prev; + Node* next = node->next; bool deleted_node = false; if (QL_NODE_IS_PLAIN(node)) { @@ -1017,12 +1013,12 @@ bool QList::Erase(const long start, unsigned count) { } Iterator it = GetIterator(start); - quicklistNode* node = it.current_; + Node* node = it.current_; long offset = it.offset_; /* iterate over next nodes until everything is deleted. */ while (extent) { - quicklistNode* next = node->next; + Node* next = node->next; unsigned long del; int delete_entire_node = 0; diff --git a/src/core/qlist.h b/src/core/qlist.h index 4ff84e2fd..2e8a85fa5 100644 --- a/src/core/qlist.h +++ b/src/core/qlist.h @@ -20,6 +20,32 @@ class QList { public: enum Where { TAIL, HEAD }; + /* Node is a 32 byte struct describing a listpack for a quicklist. + * We use bit fields keep the Node at 32 bytes. + * count: 16 bits, max 65536 (max lp bytes is 65k, so max count actually < 32k). + * encoding: 2 bits, RAW=1, LZF=2. + * container: 2 bits, PLAIN=1 (a single item as char array), PACKED=2 (listpack with multiple + * items). recompress: 1 bit, bool, true if node is temporary decompressed for usage. + * attempted_compress: 1 bit, boolean, used for verifying during testing. + * dont_compress: 1 bit, boolean, used for preventing compression of entry. + * extra: 9 bits, free for future use; pads out the remainder of 32 bits + * NOTE: do not change the ABI of this struct as long as we support --list_experimental_v2=false + * */ + + typedef struct Node { + struct Node* prev; + struct Node* next; + unsigned char* entry; + size_t sz; /* entry size in bytes */ + unsigned int count : 16; /* count of items in listpack */ + unsigned int encoding : 2; /* RAW==1 or LZF==2 */ + unsigned int container : 2; /* PLAIN==1 or PACKED==2 */ + unsigned int recompress : 1; /* was this node previous compressed? */ + unsigned int attempted_compress : 1; /* node can't compress; too small */ + unsigned int dont_compress : 1; /* prevent compression of entry that will be used later */ + unsigned int extra : 9; /* more bits to steal for future usage */ + } Node; + // Provides wrapper around the references to the listpack entries. class Entry { std::variant value_; @@ -67,7 +93,7 @@ class QList { private: const QList* owner_ = nullptr; - quicklistNode* current_ = nullptr; + Node* current_ = nullptr; unsigned char* zi_ = nullptr; /* points to the current element */ long offset_ = 0; /* offset in current listpack */ uint8_t direction_ = 1; @@ -78,8 +104,39 @@ class QList { using IterateFunc = absl::FunctionRef; enum InsertOpt { BEFORE, AFTER }; - QList(); - QList(int fill, int compress); + /** + * fill: The number of entries allowed per internal list node can be specified + * as a fixed maximum size or a maximum number of elements. + * For a fixed maximum size, use -5 through -1, meaning: + * -5: max size: 64 Kb <-- not recommended for normal workloads + * -4: max size: 32 Kb <-- not recommended + * -3: max size: 16 Kb <-- probably not recommended + * -2: max size: 8 Kb <-- good + * -1: max size: 4 Kb <-- good + * Positive numbers mean store up to _exactly_ that number of elements + * per list node. + * The highest performing option is usually -2 (8 Kb size) or -1 (4 Kb size), + * but if your use case is unique, adjust the settings as necessary. + * + * + * Lists may also be compressed. + * "compress" is the number of quicklist listpack nodes from *each* side of + * the list to *exclude* from compression. The head and tail of the list + * are always uncompressed for fast push/pop operations. Settings are: + * 0: disable all list compression + * 1: depth 1 means "don't start compressing until after 1 node into the list, + * going from either the head or tail" + * So: [head]->node->node->...->node->[tail] + * [head], [tail] will always be uncompressed; inner nodes will compress. + * 2: [head]->[next]->node->node->...->node->[prev]->[tail] + * 2 here means: don't compress head or head->next or tail->prev or tail, + * but compress all nodes between them. + * 3: [head]->[next]->[next]->node->node->...->node->[prev]->[prev]->[tail] + * etc. + * + */ + explicit QList(int fill = -2, int compress = 0); + QList(QList&&); QList(const QList&) = delete; ~QList(); @@ -139,11 +196,11 @@ class QList { bool Erase(const long start, unsigned count); // Needed by tests and the rdb code. - const quicklistNode* Head() const { + const Node* Head() const { return head_; } - const quicklistNode* Tail() const { + const Node* Tail() const { return _Tail(); } @@ -158,35 +215,37 @@ class QList { return compress_ != 0; } - quicklistNode* _Tail() const { + Node* _Tail() const { return head_ ? head_->prev : nullptr; } - void OnPreUpdate(quicklistNode* node); - void OnPostUpdate(quicklistNode* node); + void OnPreUpdate(Node* node); + void OnPostUpdate(Node* node); // Returns newly created plain node. - quicklistNode* InsertPlainNode(quicklistNode* old_node, std::string_view, InsertOpt insert_opt); - void InsertNode(quicklistNode* old_node, quicklistNode* new_node, InsertOpt insert_opt); + Node* InsertPlainNode(Node* old_node, std::string_view, InsertOpt insert_opt); + void InsertNode(Node* old_node, Node* new_node, InsertOpt insert_opt); void Replace(Iterator it, std::string_view elem); - void Compress(quicklistNode* node); + void Compress(Node* node); - quicklistNode* MergeNodes(quicklistNode* node); + Node* MergeNodes(Node* node); // Deletes one of the nodes and returns the other. - quicklistNode* ListpackMerge(quicklistNode* a, quicklistNode* b); + Node* ListpackMerge(Node* a, Node* b); - void DelNode(quicklistNode* node); - bool DelPackedIndex(quicklistNode* node, uint8_t* p); + void DelNode(Node* node); + bool DelPackedIndex(Node* node, uint8_t* p); - quicklistNode* head_ = nullptr; - size_t malloc_size_ = 0; // size of the quicklist struct - uint32_t count_ = 0; /* total count of all entries in all listpacks */ - uint32_t len_ = 0; /* number of quicklistNodes */ - signed int fill_ : QL_FILL_BITS; /* fill factor for individual nodes */ - unsigned int compress_ : QL_COMP_BITS; /* depth of end nodes not to compress;0=off */ - unsigned int bookmark_count_ : QL_BM_BITS; + Node* head_ = nullptr; + size_t malloc_size_ = 0; // size of the quicklist struct + uint32_t count_ = 0; /* total count of all entries in all listpacks */ + uint32_t len_ = 0; /* number of quicklistNodes */ + int fill_ : QL_FILL_BITS; /* fill factor for individual nodes */ + int reserved1_ : 16; + unsigned compress_ : QL_COMP_BITS; /* depth of end nodes not to compress;0=off */ + unsigned bookmark_count_ : QL_BM_BITS; + unsigned reserved2_ : 12; }; } // namespace dfly diff --git a/src/core/qlist_test.cc b/src/core/qlist_test.cc index ce7afd1a5..9da641562 100644 --- a/src/core/qlist_test.cc +++ b/src/core/qlist_test.cc @@ -28,7 +28,7 @@ static int _ql_verify_compress(const QList& ql) { int errors = 0; unsigned compress_param = ql.compress_param(); if (compress_param > 0) { - const quicklistNode* node = ql.Head(); + const auto* node = ql.Head(); unsigned int low_raw = compress_param; unsigned int high_raw = ql.node_count() - compress_param; diff --git a/src/server/rdb_save.cc b/src/server/rdb_save.cc index 2530005c4..8ae8d5289 100644 --- a/src/server/rdb_save.cc +++ b/src/server/rdb_save.cc @@ -365,7 +365,7 @@ error_code RdbSerializer::SaveListObject(const PrimeValue& pv) { } else { DCHECK_EQ(pv.Encoding(), kEncodingQL2); QList* ql = reinterpret_cast(pv.RObjPtr()); - node = ql->Head(); + node = (const quicklistNode*)ql->Head(); // We rely on common ABI for Q2 and Q1 nodes. len = ql->node_count(); } RETURN_ON_ERR(SaveLen(len));