mirrors/dragonfly
1
0
Fork 0
mirror of https://github.com/dragonflydb/dragonfly.git synced 2025-05-10 18:05:44 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

chore: remove redis sorted set implementation (#2522)

Browse source 
Also remove unused code.

Signed-off-by: Roman Gershman <roman@dragonflydb.io>
This commit is contained in:
Roman Gershman 2024-02-05 10:29:11 +02:00 • committed by GitHub
parent e0f86697f9
commit 5c0029978e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 38 additions and 1007 deletions
Download patch file Download diff file Expand all files Collapse all files

15
.github/workflows/ci.yml vendored
View file

@ -49,7 +49,7 @@ jobs:
SCCACHE_GHA_ENABLED: "true"
SCCACHE_CACHE_SIZE: 6G
SCCACHE_ERROR_LOG: /tmp/sccache_log.txt
SCCACHE_LOG: debug
# SCCACHE_LOG: debug
container:
image: ghcr.io/romange/${{ matrix.container }}
@ -128,12 +128,6 @@ jobs:
./multi_test --multi_exec_mode=1
./multi_test --multi_exec_mode=3
# GLOG_logtostderr=1 GLOG_vmodule=transaction=1,engine_shard_set=1 CTEST_OUTPUT_ON_FAILURE=1 ninja server/test
- name: Upload logs on failure
if: failure()
uses: actions/upload-artifact@v3
with:
name: unit_logs
path: /tmp/*INFO*
- name: Run regression tests
if: matrix.container == 'ubuntu-dev:20'
uses: ./.github/actions/regression-tests
@ -142,11 +136,12 @@ jobs:
run-only-on-ubuntu-latest: true
build-folder-name: build
filter: "not slow"
- name: Upload cache log
- name: Upload logs on failure
if: failure()
uses: actions/upload-artifact@v3
with:
name: sccache_log.txt
path: /tmp/sccache_log.txt
name: unit_logs
path: /tmp/*INFO*
lint-test-chart:
runs-on: ubuntu-latest

17
CMakeLists.txt
View file

@ -18,6 +18,15 @@ if (HAS_USE_AFTER_FREE_WARN)
set(CMAKE_CXX_FLAGS "-Wno-use-after-free ${CMAKE_CXX_FLAGS}")
endif()
# We can not use here CHECK_CXX_COMPILER_FLAG because systems that do not support sanitizers
# fail during linking time.
set(CMAKE_REQUIRED_FLAGS "-fsanitize=address")
check_cxx_source_compiles("int main() { return 0; }" SUPPORT_ASAN)
set(CMAKE_REQUIRED_FLAGS "-fsanitize=undefined")
check_cxx_source_compiles("int main() { return 0; }" SUPPORT_USAN)
set(CMAKE_REQUIRED_FLAGS "")
# We must define all the required variables from the root cmakefile, otherwise
# they just disappear.
set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/helio/cmake" ${CMAKE_MODULE_PATH})
@ -26,6 +35,14 @@ option(DF_USE_SSL "Provide support for SSL connections" ON)
find_package(OpenSSL)
if (SUPPORT_ASAN)
# set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address")
endif()
if (SUPPORT_USAN)
# set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=undefined")
endif()
include(third_party)
include(internal)

37
src/core/compact_object_test.cc
View file

@ -395,43 +395,6 @@ TEST_F(CompactObjectTest, Hash) {
EXPECT_EQ(1, cobj_.Size());
}
#if 0
TEST_F(CompactObjectTest, FlatSet) {
size_t allocated1, resident1, active1;
size_t allocated2, resident2, active2;
zmalloc_get_allocator_info(&allocated1, &active1, &resident1);
dict* d = dictCreate(&setDictType);
constexpr size_t kTestSize = 2000;
for (size_t i = 0; i < kTestSize; ++i) {
sds key = sdsnew("key:000000000000");
key = sdscatfmt(key, "%U", i);
dictEntry* de = dictAddRaw(d, key, NULL);
de->v.val = NULL;
}
zmalloc_get_allocator_info(&allocated2, &active2, &resident2);
size_t dict_used = allocated2 - allocated1;
dictRelease(d);
zmalloc_get_allocator_info(&allocated2, &active2, &resident2);
EXPECT_EQ(allocated2, allocated1);
MiMemoryResource mr(mi_heap_get_backing());
FlatSet fs(&mr);
for (size_t i = 0; i < kTestSize; ++i) {
string s = absl::StrCat("key:000000000000", i);
fs.Add(s);
}
zmalloc_get_allocator_info(&allocated2, &active2, &resident2);
size_t fs_used = allocated2 - allocated1;
LOG(INFO) << "dict used: " << dict_used << " fs used: " << fs_used;
EXPECT_LT(fs_used + 8 * kTestSize, dict_used);
}
#endif
TEST_F(CompactObjectTest, StreamObj) {
robj* stream_obj = createStreamObject();
stream* sm = (stream*)stream_obj->ptr;

331
src/core/sorted_map.cc
View file

@ -21,7 +21,7 @@ extern "C" {
using namespace std;
ABSL_FLAG(bool, use_zset_tree, true, "If true use b+tree for zset implementation");
ABSL_RETIRED_FLAG(bool, use_zset_tree, true, "If true use b+tree for zset implementation");
namespace dfly {
namespace detail {
@ -35,14 +35,6 @@ constexpr uint64_t kInfTag = 1ULL << 63;
constexpr uint64_t kIgnoreDoubleTag = 1ULL << 62;
constexpr uint64_t kSdsMask = (1ULL << 60) - 1;
// Approximated dictionary size.
size_t DictMallocSize(dict* d) {
size_t res = zmalloc_usable_size(d->ht_table[0]) + zmalloc_usable_size(d->ht_table[1]) +
znallocx(sizeof(dict));
return res + dictSize(d) * 16; // approximation.
}
inline zskiplistNode* Next(bool reverse, zskiplistNode* ln) {
return reverse ? ln->backward : ln->level[0].forward;
}
@ -214,319 +206,6 @@ unsigned char* ZzlInsert(unsigned char* zl, sds ele, double score) {
return zzlInsertAt(zl, NULL, ele, score);
}
void SortedMap::RdImpl::Init() {
dict = dictCreate(&zsetDictType);
zsl = zslCreate();
}
size_t SortedMap::RdImpl::MallocSize() const {
return DictMallocSize(dict) + zmalloc_size(zsl);
}
bool SortedMap::RdImpl::Insert(double score, sds member) {
zskiplistNode* znode = zslInsert(zsl, score, member);
int ret = dictAdd(dict, member, &znode->score);
return ret == DICT_OK;
}
int SortedMap::RdImpl::Add(double score, sds ele, int in_flags, int* out_flags, double* newscore) {
zskiplistNode* znode;
/* Turn options into simple to check vars. */
const bool incr = (in_flags & ZADD_IN_INCR) != 0;
const bool nx = (in_flags & ZADD_IN_NX) != 0;
const bool xx = (in_flags & ZADD_IN_XX) != 0;
const bool gt = (in_flags & ZADD_IN_GT) != 0;
const bool lt = (in_flags & ZADD_IN_LT) != 0;
*out_flags = 0; /* We'll return our response flags. */
double curscore;
dictEntry* de = dictFind(dict, ele);
if (de != NULL) {
/* NX? Return, same element already exists. */
if (nx) {
*out_flags |= ZADD_OUT_NOP;
return 1;
}
curscore = *(double*)dictGetVal(de);
/* Prepare the score for the increment if needed. */
if (incr) {
score += curscore;
if (isnan(score)) {
*out_flags |= ZADD_OUT_NAN;
return 0;
}
}
/* GT/LT? Only update if score is greater/less than current. */
if ((lt && score >= curscore) || (gt && score <= curscore)) {
*out_flags |= ZADD_OUT_NOP;
return 1;
}
*newscore = score;
/* Remove and re-insert when score changes. */
if (score != curscore) {
znode = zslUpdateScore(zsl, curscore, ele, score);
/* Note that we did not removed the original element from
* the hash table representing the sorted set, so we just
* update the score. */
dictGetVal(de) = &znode->score; /* Update score ptr. */
*out_flags |= ZADD_OUT_UPDATED;
}
return 1;
} else if (!xx) {
ele = sdsdup(ele);
znode = zslInsert(zsl, score, ele);
CHECK_EQ(DICT_OK, dictAdd(dict, ele, &znode->score));
*out_flags |= ZADD_OUT_ADDED;
*newscore = score;
return 1;
}
*out_flags |= ZADD_OUT_NOP;
return 1;
}
optional<unsigned> SortedMap::RdImpl::GetRank(sds ele, bool reverse) const {
dictEntry* de = dictFind(dict, ele);
if (de == NULL)
return std::nullopt;
double score = *(double*)dictGetVal(de);
unsigned rank = zslGetRank(zsl, score, ele);
/* Existing elements always have a rank. */
DCHECK(rank != 0);
return reverse ? zsl->length - rank : rank - 1;
}
optional<double> SortedMap::RdImpl::GetScore(sds member) const {
dictEntry* de = dictFind(dict, member);
if (de == NULL)
return std::nullopt;
return *(double*)dictGetVal(de);
}
SortedMap::ScoredArray SortedMap::RdImpl::GetRange(const zrangespec& range, unsigned offset,
unsigned limit, bool reverse) const {
/* If reversed, get the last node in range as starting point. */
zskiplistNode* ln;
if (reverse) {
ln = zslLastInRange(zsl, &range);
} else {
ln = zslFirstInRange(zsl, &range);
}
/* If there is an offset, just traverse the number of elements without
* checking the score because that is done in the next loop. */
while (ln && offset--) {
ln = Next(reverse, ln);
}
ScoredArray result;
while (ln && limit--) {
/* Abort when the node is no longer in range. */
if (!IsUnder(reverse, ln->score, range))
break;
result.emplace_back(string{ln->ele, sdslen(ln->ele)}, ln->score);
/* Move to next node */
ln = Next(reverse, ln);
}
return result;
}
SortedMap::ScoredArray SortedMap::RdImpl::GetLexRange(const zlexrangespec& range, unsigned offset,
unsigned limit, bool reverse) const {
zskiplistNode* ln;
/* If reversed, get the last node in range as starting point. */
if (reverse) {
ln = zslLastInLexRange(zsl, &range);
} else {
ln = zslFirstInLexRange(zsl, &range);
}
/* If there is an offset, just traverse the number of elements without
* checking the score because that is done in the next loop. */
while (ln && offset--) {
ln = Next(reverse, ln);
}
ScoredArray result;
while (ln && limit--) {
/* Abort when the node is no longer in range. */
if (reverse) {
if (!zslLexValueGteMin(ln->ele, &range))
break;
} else {
if (!zslLexValueLteMax(ln->ele, &range))
break;
}
result.emplace_back(string{ln->ele, sdslen(ln->ele)}, ln->score);
/* Move to next node */
ln = Next(reverse, ln);
}
return result;
}
uint8_t* SortedMap::RdImpl::ToListPack() const {
uint8_t* lp = lpNew(0);
/* Approach similar to zslFree(), since we want to free the skiplist at
* the same time as creating the listpack. */
zskiplistNode* node = zsl->header->level[0].forward;
while (node) {
lp = zzlInsertAt(lp, NULL, node->ele, node->score);
node = node->level[0].forward;
}
return lp;
}
bool SortedMap::RdImpl::Delete(sds member) {
dictEntry* de = dictUnlink(dict, member);
if (de == NULL)
return false;
/* Get the score in order to delete from the skiplist later. */
double score = *(double*)dictGetVal(de);
/* Delete from the hash table and later from the skiplist.
* Note that the order is important: deleting from the skiplist
* actually releases the SDS string representing the element,
* which is shared between the skiplist and the hash table, so
* we need to delete from the skiplist as the final step. */
dictFreeUnlinkedEntry(dict, de);
if (htNeedsResize(dict))
dictResize(dict);
/* Delete from skiplist. */
int retval = zslDelete(zsl, score, member, NULL);
DCHECK(retval);
return true;
}
SortedMap::ScoredArray SortedMap::RdImpl::PopTopScores(unsigned count, bool reverse) {
zskiplistNode* ln;
if (reverse) {
ln = zsl->tail;
} else {
ln = zsl->header->level[0].forward;
}
ScoredArray result;
while (ln && count--) {
sds ele = ln->ele;
result.emplace_back(string{ele, sdslen(ele)}, ln->score);
// Switch to next before deleting the element.
ln = Next(reverse, ln);
/* we can delete the element now */
CHECK(Delete(ele));
}
return result;
}
size_t SortedMap::RdImpl::Count(const zrangespec& range) const {
/* Find first element in range */
zskiplistNode* zn = zslFirstInRange(zsl, &range);
/* Use rank of first element, if any, to determine preliminary count */
if (zn == NULL)
return 0;
unsigned long rank = zslGetRank(zsl, zn->score, zn->ele);
size_t count = (zsl->length - (rank - 1));
/* Find last element in range */
zn = zslLastInRange(zsl, &range);
/* Use rank of last element, if any, to determine the actual count */
if (zn != NULL) {
rank = zslGetRank(zsl, zn->score, zn->ele);
count -= (zsl->length - rank);
}
return count;
}
size_t SortedMap::RdImpl::LexCount(const zlexrangespec& range) const {
/* Find first element in range */
zskiplistNode* zn = zslFirstInLexRange(zsl, &range);
/* Use rank of first element, if any, to determine preliminary count */
if (zn == NULL)
return 0;
unsigned long rank = zslGetRank(zsl, zn->score, zn->ele);
size_t count = (zsl->length - (rank - 1));
/* Find last element in range */
zn = zslLastInLexRange(zsl, &range);
/* Use rank of last element, if any, to determine the actual count */
if (zn != NULL) {
rank = zslGetRank(zsl, zn->score, zn->ele);
count -= (zsl->length - rank);
}
return count;
}
bool SortedMap::RdImpl::Iterate(unsigned start_rank, unsigned len, bool reverse,
absl::FunctionRef<bool(sds, double)> cb) const {
zskiplistNode* ln;
/* Check if starting point is trivial, before doing log(N) lookup. */
if (reverse) {
ln = zsl->tail;
unsigned long llen = zsl->length;
if (start_rank > 0)
ln = zslGetElementByRank(zsl, llen - start_rank);
} else {
ln = zsl->header->level[0].forward;
if (start_rank > 0)
ln = zslGetElementByRank(zsl, start_rank + 1);
}
bool success = true;
while (success && len--) {
DCHECK(ln != NULL);
success = cb(ln->ele, ln->score);
ln = reverse ? ln->backward : ln->level[0].forward;
if (!ln)
break;
}
return success;
}
uint64_t SortedMap::RdImpl::Scan(uint64_t cursor,
absl::FunctionRef<void(std::string_view, double)> cb) const {
auto scanCb = [](void* privdata, const dictEntry* de) {
auto* cb = reinterpret_cast<absl::FunctionRef<void(std::string_view, double)>*>(privdata);
sds key = (sds)de->key;
double score = *reinterpret_cast<double*>(dictGetVal(de));
(*cb)(std::string_view(key, sdslen(key)), score);
};
return dictScan(this->dict, cursor, scanCb, NULL, &cb);
}
int SortedMap::DfImpl::ScoreSdsPolicy::KeyCompareTo::operator()(ScoreSds a, ScoreSds b) const {
sds sdsa = (sds)(uint64_t(a) & kSdsMask);
sds sdsb = (sds)(uint64_t(b) & kSdsMask);
@ -1054,12 +733,8 @@ uint64_t SortedMap::DfImpl::Scan(uint64_t cursor,
/***************************************************************************/
/* SortedMap */
/***************************************************************************/
SortedMap::SortedMap(PMR_NS::memory_resource* mr) : impl_(RdImpl()) {
if (absl::GetFlag(FLAGS_use_zset_tree)) {
impl_ = DfImpl();
}
std::visit(Overload{[](RdImpl& impl) { impl.Init(); }, [mr](DfImpl& impl) { impl.Init(mr); }},
impl_);
SortedMap::SortedMap(PMR_NS::memory_resource* mr) : impl_(DfImpl()) {
std::visit(Overload{[mr](auto& impl) { impl.Init(mr); }}, impl_);
}
SortedMap::~SortedMap() {

64
src/core/sorted_map.h
View file

@ -13,7 +13,6 @@
#include <vector>
extern "C" {
#include "redis/dict.h"
#include "redis/zset.h"
}
@ -147,66 +146,6 @@ class SortedMap {
}
private:
struct RdImpl {
struct dict* dict = nullptr;
zskiplist* zsl = nullptr;
int Add(double score, sds ele, int in_flags, int* out_flags, double* newscore);
void Init();
void Free() {
dictRelease(dict);
zslFree(zsl);
}
bool Insert(double score, sds member);
bool Delete(sds ele);
size_t Size() const {
return zsl->length;
}
size_t MallocSize() const;
bool Reserve(size_t sz) {
return dictExpand(dict, sz) == DICT_OK;
}
size_t DeleteRangeByRank(unsigned start, unsigned end) {
return zslDeleteRangeByRank(zsl, start + 1, end + 1, dict);
}
size_t DeleteRangeByScore(const zrangespec& range) {
return zslDeleteRangeByScore(zsl, &range, dict);
}
size_t DeleteRangeByLex(const zlexrangespec& range) {
return zslDeleteRangeByLex(zsl, &range, dict);
}
ScoredArray PopTopScores(unsigned count, bool reverse);
uint8_t* ToListPack() const;
std::optional<double> GetScore(sds ele) const;
std::optional<unsigned> GetRank(sds ele, bool reverse) const;
ScoredArray GetRange(const zrangespec& r, unsigned offs, unsigned len, bool rev) const;
ScoredArray GetLexRange(const zlexrangespec& r, unsigned o, unsigned l, bool rev) const;
size_t Count(const zrangespec& range) const;
size_t LexCount(const zlexrangespec& range) const;
// Runs cb for each element in the range [start_rank, start_rank + len).
// Stops iteration if cb returns false. Returns false in this case.
bool Iterate(unsigned start_rank, unsigned len, bool reverse,
absl::FunctionRef<bool(sds, double)> cb) const;
uint64_t Scan(uint64_t cursor, absl::FunctionRef<void(std::string_view, double)> cb) const;
};
struct DfImpl {
ScoreMap* score_map = nullptr;
using ScoreSds = void*;
@ -267,7 +206,8 @@ class SortedMap {
uint64_t Scan(uint64_t cursor, absl::FunctionRef<void(std::string_view, double)> cb) const;
};
std::variant<RdImpl, DfImpl> impl_;
// TODO: remove this variant and get rid of wrapper class
std::variant<DfImpl> impl_;
};
// Used by CompactObject.

2
src/facade/dragonfly_connection.cc
View file

@ -448,7 +448,7 @@ Connection::Connection(Protocol protocol, util::HttpListenerBase* http_listener,
protocol_ = protocol;
constexpr size_t kReqSz = sizeof(Connection::PipelineMessage);
static_assert(kReqSz <= 256 && kReqSz >= 232);
static_assert(kReqSz <= 256 && kReqSz >= 200);
switch (protocol) {
case Protocol::REDIS:

2
src/facade/facade.cc
View file

@ -41,7 +41,7 @@ ConnectionStats& ConnectionStats::operator+=(const ConnectionStats& o) {
}
ReplyStats& ReplyStats::operator+=(const ReplyStats& o) {
static_assert(sizeof(ReplyStats) == 64u);
static_assert(sizeof(ReplyStats) == 64u + kSanitizerOverhead);
ADD(io_write_cnt);
ADD(io_write_bytes);

6
src/facade/facade_types.h
View file

@ -16,6 +16,12 @@
namespace facade {
#ifdef __SANITIZE_ADDRESS__
constexpr size_t kSanitizerOverhead = 24u;
#else
constexpr size_t kSanitizerOverhead = 0u;
#endif
enum class Protocol : uint8_t { MEMCACHE = 1, REDIS = 2 };
using MutableSlice = absl::Span<char>;

2
src/redis/CMakeLists.txt
View file

@ -11,7 +11,7 @@ endif()
add_library(redis_lib crc16.c crc64.c crcspeed.c debug.c dict.c intset.c geo.c
geohash.c geohash_helper.c
listpack.c mt19937-64.c object.c lzf_c.c lzf_d.c sds.c
quicklist.c rax.c pqsort.c redis_aux.c siphash.c t_hash.c t_stream.c t_zset.c
quicklist.c rax.c pqsort.c redis_aux.c siphash.c t_stream.c t_zset.c
util.c ziplist.c hyperloglog.c ${ZMALLOC_SRC})
cxx_link(redis_lib ${ZMALLOC_DEPS})

51
src/redis/object.h
View file

@ -148,61 +148,10 @@ typedef struct {
quicklistEntry entry; /* Entry in quicklist */
} listTypeEntry;
setTypeIterator *setTypeInitIterator(robj *subject);
void setTypeReleaseIterator(setTypeIterator *si);
int setTypeNext(setTypeIterator *si, sds *sdsele, int64_t *llele);
sds setTypeNextObject(setTypeIterator *si);
/* hash set interface */
/* Hash data type */
#define HASH_SET_TAKE_FIELD (1<<0)
#define HASH_SET_TAKE_VALUE (1<<1)
#define HASH_SET_COPY 0
void hashTypeConvert(robj *o, int enc);
void hashTypeTryConversion(robj *subject, robj **argv, int start, int end);
int hashTypeExists(robj *o, sds key);
int hashTypeDelete(robj *o, sds key);
unsigned long hashTypeLength(const robj *o);
hashTypeIterator *hashTypeInitIterator(robj *subject);
void hashTypeReleaseIterator(hashTypeIterator *hi);
int hashTypeNext(hashTypeIterator *hi);
void hashTypeCurrentFromListpack(hashTypeIterator *hi, int what,
unsigned char **vstr,
unsigned int *vlen,
long long *vll);
sds hashTypeCurrentFromHashTable(hashTypeIterator *hi, int what);
void hashTypeCurrentObject(hashTypeIterator *hi, int what, unsigned char **vstr, unsigned int *vlen, long long *vll);
sds hashTypeCurrentObjectNewSds(hashTypeIterator *hi, int what);
int hashTypeGetValue(robj *o, sds field, unsigned char **vstr, unsigned int *vlen, long long *vll);
robj *hashTypeGetValueObject(robj *o, sds field);
robj *hashTypeDup(robj *o);
int hashTypeGetFromListpack(robj *o, sds field,
unsigned char **vstr,
unsigned int *vlen,
long long *vll);
const char *strEncoding(int encoding);
/* Macro used to initialize a Redis object allocated on the stack.
* Note that this macro is taken near the structure definition to make sure
* we'll update it when the structure is changed, to avoid bugs like
* bug #85 introduced exactly in this way. */
#define initStaticStringObject(_var,_ptr) do { \
_var.refcount = OBJ_STATIC_REFCOUNT; \
_var.type = OBJ_STRING; \
_var.encoding = OBJ_ENCODING_RAW; \
_var.ptr = _ptr; \
} while(0)
#define serverAssertWithInfo(x, y, z) serverAssert(z)
#define PROTO_SHARED_SELECT_CMDS 10
#define OBJ_SHARED_BULKHDR_LEN 32
#endif

43
src/redis/redis_aux.c
View file

@ -30,43 +30,11 @@ void InitRedisTables() {
}
// These functions are moved here from server.c
int htNeedsResize(dict* dict) {
long long size, used;
size = dictSlots(dict);
used = dictSize(dict);
return (size > DICT_HT_INITIAL_SIZE && (used * 100 / size < HASHTABLE_MIN_FILL));
}
uint64_t dictSdsHash(const void* key) {
return dictGenHashFunction((unsigned char*)key, sdslen((char*)key));
}
// MurmurHash64A for 8 bytes blob.
uint64_t dictPtrHash(const void* key) {
const uint64_t m = 0xc6a4a7935bd1e995ULL;
const int r = 47;
uint64_t h = 120577 ^ (8 * m);
uint64_t data;
memcpy(&data, key, 8);
uint64_t k = data;
k *= m;
k ^= k >> r;
k *= m;
h ^= k;
h *= m;
h ^= h >> r;
h *= m;
h ^= h >> r;
return h;
}
int dictPtrKeyCompare(dict* privdata, const void* key1, const void* key2) {
return key1 == key2;
}
int dictSdsKeyCompare(dict* d, const void* key1, const void* key2) {
int l1, l2;
DICT_NOTUSED(d);
@ -127,17 +95,6 @@ dictType setDictType = {
NULL /* allow to expand */
};
/* Sorted sets hash (note: a skiplist is used in addition to the hash table) */
dictType zsetDictType = {
dictSdsHash, /* hash function */
NULL, /* key dup */
NULL, /* val dup */
dictSdsKeyCompare, /* key compare */
NULL, /* Note: SDS string shared & freed by skiplist */
NULL, /* val destructor */
NULL /* allow to expand */
};
/* Hash type hash table (note that small hashes are represented with listpacks) */
dictType hashDictType = {
dictSdsHash, /* hash function */

12
src/redis/redis_aux.h
View file

@ -36,13 +36,7 @@
#define CONFIG_RUN_ID_SIZE 40U
#define EVPOOL_CACHED_SDS_SIZE 255
#define EVPOOL_SIZE 16
int htNeedsResize(dict* dict); // moved from server.cc
/* Hash table types */
extern dictType zsetDictType;
extern dictType setDictType;
extern dictType hashDictType;
@ -58,12 +52,6 @@ extern dictType hashDictType;
*
* Empty entries have the key pointer set to NULL. */
struct evictionPoolEntry {
unsigned long long idle; /* Object idle time (inverse frequency for LFU) */
sds key; /* Key name. */
sds cached; /* Cached SDS object for key name. */
int dbid; /* Key DB number. */
};
uint64_t dictSdsHash(const void* key);
int dictSdsKeyCompare(dict* privdata, const void* key1, const void* key2);

460
src/redis/t_hash.c
View file

@ -1,460 +0,0 @@
/*
* Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <string.h>
// ROMAN - taken from redis 7.0 branch.
#include "listpack.h"
#include "object.h"
#include "redis_aux.h"
#include "util.h"
#include "zmalloc.h"
/*-----------------------------------------------------------------------------
* Hash type API
*----------------------------------------------------------------------------*/
/* Get the value from a listpack encoded hash, identified by field.
* Returns -1 when the field cannot be found. */
int hashTypeGetFromListpack(robj *o, sds field,
unsigned char **vstr,
unsigned int *vlen,
long long *vll)
{
unsigned char *zl, *fptr = NULL, *vptr = NULL;
serverAssert(o->encoding == OBJ_ENCODING_LISTPACK);
zl = o->ptr;
fptr = lpFirst(zl);
if (fptr != NULL) {
fptr = lpFind(zl, fptr, (unsigned char*)field, sdslen(field), 1);
if (fptr != NULL) {
/* Grab pointer to the value (fptr points to the field) */
vptr = lpNext(zl, fptr);
serverAssert(vptr != NULL);
}
}
if (vptr != NULL) {
*vstr = lpGetValue(vptr, vlen, vll);
return 0;
}
return -1;
}
/* Get the value from a hash table encoded hash, identified by field.
* Returns NULL when the field cannot be found, otherwise the SDS value
* is returned. */
sds hashTypeGetFromHashTable(robj *o, sds field) {
dictEntry *de;
serverAssert(o->encoding == OBJ_ENCODING_HT);
de = dictFind(o->ptr, field);
if (de == NULL) return NULL;
return dictGetVal(de);
}
/* Higher level function of hashTypeGet*() that returns the hash value
* associated with the specified field. If the field is found C_OK
* is returned, otherwise C_ERR. The returned object is returned by
* reference in either *vstr and *vlen if it's returned in string form,
* or stored in *vll if it's returned as a number.
*
* If *vll is populated *vstr is set to NULL, so the caller
* can always check the function return by checking the return value
* for C_OK and checking if vll (or vstr) is NULL. */
int hashTypeGetValue(robj *o, sds field, unsigned char **vstr, unsigned int *vlen, long long *vll) {
if (o->encoding == OBJ_ENCODING_LISTPACK) {
*vstr = NULL;
if (hashTypeGetFromListpack(o, field, vstr, vlen, vll) == 0)
return C_OK;
} else if (o->encoding == OBJ_ENCODING_HT) {
sds value;
if ((value = hashTypeGetFromHashTable(o, field)) != NULL) {
*vstr = (unsigned char*) value;
*vlen = sdslen(value);
return C_OK;
}
} else {
serverPanic("Unknown hash encoding");
}
return C_ERR;
}
/* Like hashTypeGetValue() but returns a Redis object, which is useful for
* interaction with the hash type outside t_hash.c.
* The function returns NULL if the field is not found in the hash. Otherwise
* a newly allocated string object with the value is returned. */
robj *hashTypeGetValueObject(robj *o, sds field) {
unsigned char *vstr;
unsigned int vlen;
long long vll;
if (hashTypeGetValue(o,field,&vstr,&vlen,&vll) == C_ERR) return NULL;
if (vstr) return createStringObject((char*)vstr,vlen);
else return createStringObjectFromLongLong(vll);
}
/* Higher level function using hashTypeGet*() to return the length of the
* object associated with the requested field, or 0 if the field does not
* exist. */
size_t hashTypeGetValueLength(robj *o, sds field) {
size_t len = 0;
if (o->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *vstr = NULL;
unsigned int vlen = UINT_MAX;
long long vll = LLONG_MAX;
if (hashTypeGetFromListpack(o, field, &vstr, &vlen, &vll) == 0)
len = vstr ? vlen : sdigits10(vll);
} else if (o->encoding == OBJ_ENCODING_HT) {
sds aux;
if ((aux = hashTypeGetFromHashTable(o, field)) != NULL)
len = sdslen(aux);
} else {
serverPanic("Unknown hash encoding");
}
return len;
}
/* Test if the specified field exists in the given hash. Returns 1 if the field
* exists, and 0 when it doesn't. */
int hashTypeExists(robj *o, sds field) {
if (o->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *vstr = NULL;
unsigned int vlen = UINT_MAX;
long long vll = LLONG_MAX;
if (hashTypeGetFromListpack(o, field, &vstr, &vlen, &vll) == 0) return 1;
} else if (o->encoding == OBJ_ENCODING_HT) {
if (hashTypeGetFromHashTable(o, field) != NULL) return 1;
} else {
serverPanic("Unknown hash encoding");
}
return 0;
}
/* Delete an element from a hash.
* Return 1 on deleted and 0 on not found. */
int hashTypeDelete(robj *o, sds field) {
int deleted = 0;
if (o->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *zl, *fptr;
zl = o->ptr;
fptr = lpFirst(zl);
if (fptr != NULL) {
fptr = lpFind(zl, fptr, (unsigned char*)field, sdslen(field), 1);
if (fptr != NULL) {
/* Delete both of the key and the value. */
zl = lpDeleteRangeWithEntry(zl,&fptr,2);
o->ptr = zl;
deleted = 1;
}
}
} else if (o->encoding == OBJ_ENCODING_HT) {
if (dictDelete((dict*)o->ptr, field) == C_OK) {
deleted = 1;
/* Always check if the dictionary needs a resize after a delete. */
if (htNeedsResize(o->ptr)) dictResize(o->ptr);
}
} else {
serverPanic("Unknown hash encoding");
}
return deleted;
}
/* Return the number of elements in a hash. */
unsigned long hashTypeLength(const robj *o) {
unsigned long length = ULONG_MAX;
if (o->encoding == OBJ_ENCODING_LISTPACK) {
length = lpLength(o->ptr) / 2;
} else if (o->encoding == OBJ_ENCODING_HT) {
length = dictSize((const dict*)o->ptr);
} else {
serverPanic("Unknown hash encoding");
}
return length;
}
hashTypeIterator *hashTypeInitIterator(robj *subject) {
hashTypeIterator *hi = zmalloc(sizeof(hashTypeIterator));
hi->subject = subject;
hi->encoding = subject->encoding;
if (hi->encoding == OBJ_ENCODING_LISTPACK) {
hi->fptr = NULL;
hi->vptr = NULL;
} else if (hi->encoding == OBJ_ENCODING_HT) {
hi->di = dictGetIterator(subject->ptr);
} else {
serverPanic("Unknown hash encoding");
}
return hi;
}
void hashTypeReleaseIterator(hashTypeIterator *hi) {
if (hi->encoding == OBJ_ENCODING_HT)
dictReleaseIterator(hi->di);
zfree(hi);
}
/* Move to the next entry in the hash. Return C_OK when the next entry
* could be found and C_ERR when the iterator reaches the end. */
int hashTypeNext(hashTypeIterator *hi) {
if (hi->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *zl;
unsigned char *fptr, *vptr;
zl = hi->subject->ptr;
fptr = hi->fptr;
vptr = hi->vptr;
if (fptr == NULL) {
/* Initialize cursor */
serverAssert(vptr == NULL);
fptr = lpFirst(zl);
} else {
/* Advance cursor */
serverAssert(vptr != NULL);
fptr = lpNext(zl, vptr);
}
if (fptr == NULL) return C_ERR;
/* Grab pointer to the value (fptr points to the field) */
vptr = lpNext(zl, fptr);
serverAssert(vptr != NULL);
/* fptr, vptr now point to the first or next pair */
hi->fptr = fptr;
hi->vptr = vptr;
} else if (hi->encoding == OBJ_ENCODING_HT) {
if ((hi->de = dictNext(hi->di)) == NULL) return C_ERR;
} else {
serverPanic("Unknown hash encoding");
}
return C_OK;
}
/* Get the field or value at iterator cursor, for an iterator on a hash value
* encoded as a listpack. Prototype is similar to `hashTypeGetFromListpack`. */
void hashTypeCurrentFromListpack(hashTypeIterator *hi, int what,
unsigned char **vstr,
unsigned int *vlen,
long long *vll)
{
serverAssert(hi->encoding == OBJ_ENCODING_LISTPACK);
if (what & OBJ_HASH_KEY) {
*vstr = lpGetValue(hi->fptr, vlen, vll);
} else {
*vstr = lpGetValue(hi->vptr, vlen, vll);
}
}
/* Get the field or value at iterator cursor, for an iterator on a hash value
* encoded as a hash table. Prototype is similar to
* `hashTypeGetFromHashTable`. */
sds hashTypeCurrentFromHashTable(hashTypeIterator *hi, int what) {
serverAssert(hi->encoding == OBJ_ENCODING_HT);
if (what & OBJ_HASH_KEY) {
return dictGetKey(hi->de);
} else {
return dictGetVal(hi->de);
}
}
/* Higher level function of hashTypeCurrent*() that returns the hash value
* at current iterator position.
*
* The returned element is returned by reference in either *vstr and *vlen if
* it's returned in string form, or stored in *vll if it's returned as
* a number.
*
* If *vll is populated *vstr is set to NULL, so the caller
* can always check the function return by checking the return value
* type checking if vstr == NULL. */
void hashTypeCurrentObject(hashTypeIterator *hi, int what, unsigned char **vstr, unsigned int *vlen, long long *vll) {
if (hi->encoding == OBJ_ENCODING_LISTPACK) {
*vstr = NULL;
hashTypeCurrentFromListpack(hi, what, vstr, vlen, vll);
} else if (hi->encoding == OBJ_ENCODING_HT) {
sds ele = hashTypeCurrentFromHashTable(hi, what);
*vstr = (unsigned char*) ele;
*vlen = sdslen(ele);
} else {
serverPanic("Unknown hash encoding");
}
}
/* Return the key or value at the current iterator position as a new
* SDS string. */
sds hashTypeCurrentObjectNewSds(hashTypeIterator *hi, int what) {
unsigned char *vstr;
unsigned int vlen;
long long vll;
hashTypeCurrentObject(hi,what,&vstr,&vlen,&vll);
if (vstr) return sdsnewlen(vstr,vlen);
return sdsfromlonglong(vll);
}
void hashTypeConvertListpack(robj *o, int enc) {
serverAssert(o->encoding == OBJ_ENCODING_LISTPACK);
if (enc == OBJ_ENCODING_LISTPACK) {
/* Nothing to do... */
} else if (enc == OBJ_ENCODING_HT) {
hashTypeIterator *hi;
dict *dict;
int ret;
hi = hashTypeInitIterator(o);
dict = dictCreate(&hashDictType);
/* Presize the dict to avoid rehashing */
dictExpand(dict,hashTypeLength(o));
while (hashTypeNext(hi) != C_ERR) {
sds key, value;
key = hashTypeCurrentObjectNewSds(hi,OBJ_HASH_KEY);
value = hashTypeCurrentObjectNewSds(hi,OBJ_HASH_VALUE);
ret = dictAdd(dict, key, value);
if (ret != DICT_OK) {
sdsfree(key); sdsfree(value); /* Needed for gcc ASAN */
hashTypeReleaseIterator(hi); /* Needed for gcc ASAN */
serverLogHexDump(LL_WARNING,"listpack with dup elements dump",
o->ptr,lpBytes(o->ptr));
serverPanic("Listpack corruption detected");
}
}
hashTypeReleaseIterator(hi);
zfree(o->ptr);
o->encoding = OBJ_ENCODING_HT;
o->ptr = dict;
} else {
serverPanic("Unknown hash encoding");
}
}
void hashTypeConvert(robj *o, int enc) {
if (o->encoding == OBJ_ENCODING_LISTPACK) {
hashTypeConvertListpack(o, enc);
} else if (o->encoding == OBJ_ENCODING_HT) {
serverPanic("Not implemented");
} else {
serverPanic("Unknown hash encoding");
}
}
/* This is a helper function for the COPY command.
* Duplicate a hash object, with the guarantee that the returned object
* has the same encoding as the original one.
*
* The resulting object always has refcount set to 1 */
robj *hashTypeDup(robj *o) {
robj *hobj;
hashTypeIterator *hi;
serverAssert(o->type == OBJ_HASH);
if(o->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *zl = o->ptr;
size_t sz = lpBytes(zl);
unsigned char *new_zl = zmalloc(sz);
memcpy(new_zl, zl, sz);
hobj = createObject(OBJ_HASH, new_zl);
hobj->encoding = OBJ_ENCODING_LISTPACK;
} else if(o->encoding == OBJ_ENCODING_HT){
dict *d = dictCreate(&hashDictType);
dictExpand(d, dictSize((const dict*)o->ptr));
hi = hashTypeInitIterator(o);
while (hashTypeNext(hi) != C_ERR) {
sds field, value;
sds newfield, newvalue;
/* Extract a field-value pair from an original hash object.*/
field = hashTypeCurrentFromHashTable(hi, OBJ_HASH_KEY);
value = hashTypeCurrentFromHashTable(hi, OBJ_HASH_VALUE);
newfield = sdsdup(field);
newvalue = sdsdup(value);
/* Add a field-value pair to a new hash object. */
dictAdd(d,newfield,newvalue);
}
hashTypeReleaseIterator(hi);
hobj = createObject(OBJ_HASH, d);
hobj->encoding = OBJ_ENCODING_HT;
} else {
serverPanic("Unknown hash encoding");
}
return hobj;
}
/* Create a new sds string from the listpack entry. */
sds hashSdsFromListpackEntry(listpackEntry *e) {
return e->sval ? sdsnewlen(e->sval, e->slen) : sdsfromlonglong(e->lval);
}
/* Return random element from a non empty hash.
* 'key' and 'val' will be set to hold the element.
* The memory in them is not to be freed or modified by the caller.
* 'val' can be NULL in which case it's not extracted. */
void hashTypeRandomElement(robj *hashobj, unsigned long hashsize, listpackEntry *key, listpackEntry *val) {
if (hashobj->encoding == OBJ_ENCODING_HT) {
dictEntry *de = dictGetFairRandomKey(hashobj->ptr);
sds s = dictGetKey(de);
key->sval = (unsigned char*)s;
key->slen = sdslen(s);
if (val) {
sds s = dictGetVal(de);
val->sval = (unsigned char*)s;
val->slen = sdslen(s);
}
} else if (hashobj->encoding == OBJ_ENCODING_LISTPACK) {
lpRandomPair(hashobj->ptr, hashsize, key, val);
} else {
serverPanic("Unknown hash encoding");
}
}

2
src/server/list_family_test.cc
View file

@ -863,6 +863,8 @@ TEST_F(ListFamilyTest, OtherMultiWakesBLpop) {
}
TEST_F(ListFamilyTest, ContendExpire) {
GTEST_SKIP() << "Skipped due to a bug in helio";
vector<fb2::Fiber> blpop_fibers;
for (unsigned i = 0; i < num_threads_; ++i) {
for (unsigned j = 0; j < 30; ++j) {

1
tests/dragonfly/instance.py
View file

@ -318,7 +318,6 @@ class DflyInstanceFactory:
def create(self, existing_port=None, **kwargs) -> DflyInstance:
args = {**self.args, **kwargs}
args.setdefault("dbfilename", "")
args.setdefault("use_zset_tree", None)
vmod = "dragonfly_connection=1,accept_server=1,listener_interface=1,main_service=1,rdb_save=1,replica=1"
args.setdefault("vmodule", vmod)