chore: fix some of FreeBsd compile issues. (#1374)

1. Use experimental/memory_resource because clang on freebsd does not support yet std::memory_resource
2. Pull latest helio with all the compatibility fixes.
3. Replace linux only SOL_TCP constant with IPPROTO_TCP (same value).
4. Fix build files to work on FreeBsd system.

Signed-off-by: Roman Gershman <roman@dragonflydb.io>
This commit is contained in:
Roman Gershman 2023-06-11 23:22:56 +03:00 committed by GitHub
parent 9c8b3296b2
commit cf1ac87156
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 105 additions and 59 deletions

2
helio

@ -1 +1 @@
Subproject commit 724eb6441516fe07517ec1841e96e6b68f6ff183
Subproject commit a48b0447c711337451d66b0bc4f5bfe6be6913d2

View file

@ -15,25 +15,27 @@ diff --git a/makefile b/makefile
index d46e650c..e347e614 100644
--- a/makefile
+++ b/makefile
@@ -66,13 +66,21 @@ LOCAL = $(TESTS) $(CWARNS)
@@ -66,13 +66,23 @@ LOCAL = $(TESTS) $(CWARNS)
# enable Linux goodies
-MYCFLAGS= $(LOCAL) -std=c99 -DLUA_USE_LINUX -DLUA_USE_READLINE
+MYCFLAGS= $(LOCAL) -std=c99 -g -O2 -DLUA_USE_LINUX
MYLDFLAGS= $(LOCAL) -Wl,-E
-MYLIBS= -ldl -lreadline
+MYLIBS= -ldl
+uname_m := $(shell uname -m)
+ifeq ($(uname_m),x86_64)
+
+# equivalent to: if $(uname_m) == x86_64 || $(uname_m) == amd64
+ifneq (, $(uname_m),x86_64,amd64)
+OPTFLAGS= -march=sandybridge
+else ifeq ($(uname_m), aarch64)
+OPTFLAGS= -march=armv8.2-a+fp16+rcpc+dotprod+crypto
+else
+ $(error ERROR: unknown architecture $(uname_m))
+endif
CC= gcc
-CFLAGS= -Wall -O2 $(MYCFLAGS) -fno-stack-protector -fno-common -march=native
+CFLAGS= -Wall -O2 $(MYCFLAGS) -fno-stack-protector -fno-common $(OPTFLAGS)

View file

@ -1,9 +1,17 @@
if ("${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD")
set(DFLY_TOOLS_MAKE "gmake")
else()
set(DFLY_TOOLS_MAKE "make")
endif()
add_third_party(
lua
URL https://github.com/lua/lua/archive/refs/tags/v5.4.4.tar.gz
PATCH_COMMAND patch -p1 -i "${CMAKE_SOURCE_DIR}/patches/lua-v5.4.4.patch"
CONFIGURE_COMMAND echo
BUILD_IN_SOURCE 1
BUILD_COMMAND ${DFLY_TOOLS_MAKE} all
INSTALL_COMMAND cp <SOURCE_DIR>/liblua.a ${THIRD_PARTY_LIB_DIR}/lua/lib/
COMMAND cp <SOURCE_DIR>/lualib.h <SOURCE_DIR>/lua.h <SOURCE_DIR>/lauxlib.h
<SOURCE_DIR>/luaconf.h ${THIRD_PARTY_LIB_DIR}/lua/include
@ -64,8 +72,8 @@ add_third_party(
BUILD_IN_SOURCE 1
CONFIGURE_COMMAND echo skip
BUILD_COMMAND make lib-release
INSTALL_COMMAND make install BUILD_SHARED=no PREFIX=${THIRD_PARTY_LIB_DIR}/lz4
BUILD_COMMAND ${DFLY_TOOLS_MAKE} lib-release
INSTALL_COMMAND ${DFLY_TOOLS_MAKE} install BUILD_SHARED=no PREFIX=${THIRD_PARTY_LIB_DIR}/lz4
)
add_library(TRDP::jsoncons INTERFACE IMPORTED)

View file

@ -33,7 +33,7 @@ namespace dfly {
using namespace std;
using absl::GetFlag;
using detail::binpacked_len;
using MemoryResource = detail::RobjWrapper::MemoryResource;
namespace {
constexpr XXH64_hash_t kHashSeed = 24061983;
@ -53,7 +53,7 @@ size_t DictMallocSize(dict* d) {
return res + dictSize(d) * 16; // approximation.
}
inline void FreeObjSet(unsigned encoding, void* ptr, pmr::memory_resource* mr) {
inline void FreeObjSet(unsigned encoding, void* ptr, MemoryResource* mr) {
switch (encoding) {
case kEncodingStrMap: {
dictRelease((dict*)ptr);
@ -181,7 +181,7 @@ struct TL {
robj tmp_robj{
.type = 0, .encoding = 0, .lru = 0, .refcount = OBJ_STATIC_REFCOUNT, .ptr = nullptr};
pmr::memory_resource* local_mr = pmr::get_default_resource();
MemoryResource* local_mr = PMR_NS::get_default_resource();
size_t small_str_bytes;
base::PODArray<uint8_t> tmp_buf;
string tmp_str;
@ -266,7 +266,7 @@ size_t RobjWrapper::Size() const {
return 0;
}
void RobjWrapper::Free(pmr::memory_resource* mr) {
void RobjWrapper::Free(MemoryResource* mr) {
if (!inner_obj_)
return;
DVLOG(1) << "RobjWrapper::Free " << inner_obj_;
@ -338,7 +338,7 @@ bool RobjWrapper::Equal(string_view sv) const {
return AsView() == sv;
}
void RobjWrapper::SetString(string_view s, pmr::memory_resource* mr) {
void RobjWrapper::SetString(string_view s, MemoryResource* mr) {
type_ = OBJ_STRING;
encoding_ = OBJ_ENCODING_RAW;
@ -361,7 +361,7 @@ bool RobjWrapper::DefragIfNeeded(float ratio) {
return false;
}
bool RobjWrapper::Reallocate(std::pmr::memory_resource* mr) {
bool RobjWrapper::Reallocate(MemoryResource* mr) {
void* old_ptr = inner_obj_;
inner_obj_ = mr->allocate(sz_, kAlignSize);
memcpy(inner_obj_, old_ptr, sz_);
@ -379,7 +379,7 @@ inline size_t RobjWrapper::InnerObjMallocUsed() const {
return zmalloc_size(inner_obj_);
}
void RobjWrapper::MakeInnerRoom(size_t current_cap, size_t desired, pmr::memory_resource* mr) {
void RobjWrapper::MakeInnerRoom(size_t current_cap, size_t desired, MemoryResource* mr) {
if (current_cap * 2 > desired) {
if (desired < SDS_MAX_PREALLOC)
desired *= 2;
@ -409,7 +409,7 @@ auto CompactObj::GetStats() -> Stats {
return res;
}
void CompactObj::InitThreadLocal(pmr::memory_resource* mr) {
void CompactObj::InitThreadLocal(MemoryResource* mr) {
tl.local_mr = mr;
tl.tmp_buf = base::PODArray<uint8_t>{mr};
}
@ -1057,7 +1057,7 @@ size_t CompactObj::DecodedLen(size_t sz) const {
return ascii_len(sz) - ((mask_ & ASCII1_ENC_BIT) ? 1 : 0);
}
pmr::memory_resource* CompactObj::memory_resource() {
MemoryResource* CompactObj::memory_resource() {
return tl.local_mr;
}

View file

@ -6,7 +6,14 @@
#include <absl/base/internal/endian.h>
#ifdef __clang__
#include <experimental/memory_resource>
namespace PMR_NS = std::experimental::pmr;
#else
#include <memory_resource>
namespace PMR_NS = std::pmr;
#endif
#include <optional>
#include "core/json_object.h"
@ -26,6 +33,8 @@ namespace detail {
// redis objects or blobs of upto 4GB size.
class RobjWrapper {
public:
using MemoryResource = PMR_NS::memory_resource;
RobjWrapper() {
}
size_t MallocUsed() const;
@ -34,9 +43,9 @@ class RobjWrapper {
bool Equal(const RobjWrapper& ow) const;
bool Equal(std::string_view sv) const;
size_t Size() const;
void Free(std::pmr::memory_resource* mr);
void Free(MemoryResource* mr);
void SetString(std::string_view s, std::pmr::memory_resource* mr);
void SetString(std::string_view s, MemoryResource* mr);
void Init(unsigned type, unsigned encoding, void* inner);
unsigned type() const {
@ -56,9 +65,9 @@ class RobjWrapper {
bool DefragIfNeeded(float ratio);
private:
bool Reallocate(std::pmr::memory_resource* mr);
bool Reallocate(MemoryResource* mr);
size_t InnerObjMallocUsed() const;
void MakeInnerRoom(size_t current_cap, size_t desired, std::pmr::memory_resource* mr);
void MakeInnerRoom(size_t current_cap, size_t desired, MemoryResource* mr);
void Set(void* p, uint32_t s) {
inner_obj_ = p;
@ -106,6 +115,7 @@ class CompactObj {
public:
using PrefixArray = std::vector<std::string_view>;
using MemoryResource = detail::RobjWrapper::MemoryResource;
CompactObj() { // By default - empty string.
}
@ -291,8 +301,8 @@ class CompactObj {
static Stats GetStats();
static void InitThreadLocal(std::pmr::memory_resource* mr);
static std::pmr::memory_resource* memory_resource(); // thread-local.
static void InitThreadLocal(MemoryResource* mr);
static MemoryResource* memory_resource(); // thread-local.
private:
size_t DecodedLen(size_t sz) const;
@ -408,3 +418,5 @@ class CompactObjectView {
};
} // namespace dfly
#undef PMR_NS

View file

@ -87,7 +87,7 @@ class CompactObjectTest : public ::testing::Test {
auto* tlh = mi_heap_get_backing();
init_zmalloc_threadlocal(tlh);
SmallString::InitThreadLocal(tlh);
CompactObj::InitThreadLocal(pmr::get_default_resource());
CompactObj::InitThreadLocal(PMR_NS::get_default_resource());
}
static void TearDownTestSuite() {
@ -384,6 +384,7 @@ TEST_F(CompactObjectTest, ZSet) {
EXPECT_EQ(OBJ_ENCODING_LISTPACK, cobj_.Encoding());
}
#if 0
TEST_F(CompactObjectTest, FlatSet) {
size_t allocated1, resident1, active1;
size_t allocated2, resident2, active2;
@ -418,6 +419,7 @@ TEST_F(CompactObjectTest, FlatSet) {
LOG(INFO) << "dict used: " << dict_used << " fs used: " << fs_used;
EXPECT_LT(fs_used + 8 * kTestSize, dict_used);
}
#endif
TEST_F(CompactObjectTest, StreamObj) {
robj* stream_obj = createStreamObject();

View file

@ -3,7 +3,14 @@
//
#pragma once
#ifdef __clang__
#include <experimental/memory_resource>
namespace PMR_NS = std::experimental::pmr;
#else
#include <memory_resource>
namespace PMR_NS = std::pmr;
#endif
#include <vector>
#include "core/dash_internal.h"
@ -127,7 +134,7 @@ class DashTable : public detail::DashTableBase {
};
DashTable(size_t capacity_log = 1, const Policy& policy = Policy{},
std::pmr::memory_resource* mr = std::pmr::get_default_resource());
PMR_NS::memory_resource* mr = PMR_NS::get_default_resource());
~DashTable();
void Reserve(size_t size);
@ -291,7 +298,7 @@ class DashTable : public detail::DashTableBase {
}
Policy policy_;
std::pmr::vector<SegmentType*> segment_;
std::vector<SegmentType*, PMR_NS::polymorphic_allocator<SegmentType*>> segment_;
uint64_t garbage_collected_ = 0;
uint64_t stash_unloaded_ = 0;
@ -474,10 +481,10 @@ void DashTable<_Key, _Value, Policy>::Iterator<IsConst, IsSingleBucket>::Seek2Oc
template <typename _Key, typename _Value, typename Policy>
DashTable<_Key, _Value, Policy>::DashTable(size_t capacity_log, const Policy& policy,
std::pmr::memory_resource* mr)
PMR_NS::memory_resource* mr)
: Base(capacity_log), policy_(policy), segment_(mr) {
segment_.resize(unique_segments_);
std::pmr::polymorphic_allocator<SegmentType> pa(mr);
PMR_NS::polymorphic_allocator<SegmentType> pa(mr);
// I assume we have enough memory to create the initial table and do not check allocations.
for (auto& ptr : segment_) {
@ -490,7 +497,7 @@ template <typename _Key, typename _Value, typename Policy>
DashTable<_Key, _Value, Policy>::~DashTable() {
Clear();
auto* resource = segment_.get_allocator().resource();
std::pmr::polymorphic_allocator<SegmentType> pa(resource);
PMR_NS::polymorphic_allocator<SegmentType> pa(resource);
using alloc_traits = std::allocator_traits<decltype(pa)>;
IterateDistinct([&](SegmentType* seg) {
@ -574,7 +581,7 @@ void DashTable<_Key, _Value, Policy>::Clear() {
and then erase all other distinct segments.
**********/
if (global_depth_ > initial_depth_) {
std::pmr::polymorphic_allocator<SegmentType> pa(segment_.get_allocator());
PMR_NS::polymorphic_allocator<SegmentType> pa(segment_.get_allocator());
using alloc_traits = std::allocator_traits<decltype(pa)>;
size_t dest = 0, src = 0;
@ -837,7 +844,7 @@ void DashTable<_Key, _Value, Policy>::Split(uint32_t seg_id) {
size_t chunk_size = 1u << (global_depth_ - source->local_depth());
size_t start_idx = seg_id & (~(chunk_size - 1));
assert(segment_[start_idx] == source && segment_[start_idx + chunk_size - 1] == source);
std::pmr::polymorphic_allocator<SegmentType> alloc(segment_.get_allocator().resource());
PMR_NS::polymorphic_allocator<SegmentType> alloc(segment_.get_allocator().resource());
SegmentType* target = alloc.allocate(1);
alloc.construct(target, source->local_depth() + 1);

View file

@ -83,7 +83,7 @@ struct RelaxedBumpPolicy {
}
};
class CappedResource final : public std::pmr::memory_resource {
class CappedResource final : public PMR_NS::memory_resource {
public:
explicit CappedResource(size_t cap) : cap_(cap) {
}
@ -97,7 +97,7 @@ class CappedResource final : public std::pmr::memory_resource {
if (used_ + size > cap_)
throw std::bad_alloc{};
void* res = pmr::get_default_resource()->allocate(size, align);
void* res = PMR_NS::get_default_resource()->allocate(size, align);
used_ += size;
return res;
@ -105,10 +105,10 @@ class CappedResource final : public std::pmr::memory_resource {
void do_deallocate(void* ptr, std::size_t size, std::size_t align) {
used_ -= size;
pmr::get_default_resource()->deallocate(ptr, size, align);
PMR_NS::get_default_resource()->deallocate(ptr, size, align);
}
bool do_is_equal(const std::pmr::memory_resource& o) const noexcept {
bool do_is_equal(const PMR_NS::memory_resource& o) const noexcept {
return this == &o;
}
@ -804,7 +804,6 @@ struct BlankPolicy : public BasicDashPolicy {
}
};
// The bug was that for very rare cases when during segment splitting we move all the items
// into a new segment, not every item finds a place.
TEST_F(DashTest, SplitBug) {

View file

@ -67,7 +67,7 @@ void DenseSet::IteratorBase::Advance() {
DCHECK(!curr_entry_->IsEmpty());
}
DenseSet::DenseSet(pmr::memory_resource* mr) : entries_(mr) {
DenseSet::DenseSet(MemoryResource* mr) : entries_(mr) {
}
DenseSet::~DenseSet() {
@ -466,7 +466,7 @@ void DenseSet::Delete(DensePtr* prev, DensePtr* ptr) {
}
void* DenseSet::PopInternal() {
std::pmr::vector<DenseSet::DensePtr>::iterator bucket_iter = entries_.begin();
ChainVectorIterator bucket_iter = entries_.begin();
// find the first non-empty chain
do {

View file

@ -6,9 +6,16 @@
#include <cstddef>
#include <cstdint>
#include <functional>
#include <memory_resource>
#include <type_traits>
#ifdef __clang__
#include <experimental/memory_resource>
namespace PMR_NS = std::experimental::pmr;
#else
#include <memory_resource>
namespace PMR_NS = std::pmr;
#endif
namespace dfly {
// DenseSet is a nice but over-optimized data-structure. Probably is not worth it in the first
@ -166,9 +173,10 @@ class DenseSet {
static_assert(sizeof(DenseLinkKey) == 2 * sizeof(uintptr_t));
protected:
using LinkAllocator = std::pmr::polymorphic_allocator<DenseLinkKey>;
using ChainVectorIterator = std::pmr::vector<DensePtr>::iterator;
using ChainVectorConstIterator = std::pmr::vector<DensePtr>::const_iterator;
using LinkAllocator = PMR_NS::polymorphic_allocator<DenseLinkKey>;
using DensePtrAllocator = PMR_NS::polymorphic_allocator<DensePtr>;
using ChainVectorIterator = std::vector<DensePtr, DensePtrAllocator>::iterator;
using ChainVectorConstIterator = std::vector<DensePtr, DensePtrAllocator>::const_iterator;
class IteratorBase {
protected:
@ -185,7 +193,9 @@ class DenseSet {
};
public:
explicit DenseSet(std::pmr::memory_resource* mr = std::pmr::get_default_resource());
using MemoryResource = PMR_NS::memory_resource;
explicit DenseSet(MemoryResource* mr = PMR_NS::get_default_resource());
virtual ~DenseSet();
size_t Size() const {
@ -281,7 +291,7 @@ class DenseSet {
bool Equal(DensePtr dptr, const void* ptr, uint32_t cookie) const;
std::pmr::memory_resource* mr() {
MemoryResource* mr() {
return entries_.get_allocator().resource();
}
@ -330,7 +340,7 @@ class DenseSet {
// If ptr is a link then it will be deleted internally.
void Delete(DensePtr* prev, DensePtr* ptr);
std::pmr::vector<DensePtr> entries_;
std::vector<DensePtr, DensePtrAllocator> entries_;
mutable size_t obj_malloc_used_ = 0;
mutable uint32_t size_ = 0;

View file

@ -6,11 +6,17 @@
#include <mimalloc.h>
#ifdef __clang__
#include <experimental/memory_resource>
namespace PMR_NS = std::experimental::pmr;
#else
#include <memory_resource>
namespace PMR_NS = std::pmr;
#endif
namespace dfly {
class MiMemoryResource : public std::pmr::memory_resource {
class MiMemoryResource : public PMR_NS::memory_resource {
public:
explicit MiMemoryResource(mi_heap_t* heap) : heap_(heap) {
}
@ -28,7 +34,7 @@ class MiMemoryResource : public std::pmr::memory_resource {
void do_deallocate(void* ptr, std::size_t size, std::size_t align) final;
bool do_is_equal(const std::pmr::memory_resource& o) const noexcept {
bool do_is_equal(const PMR_NS::memory_resource& o) const noexcept {
return this == &o;
}

View file

@ -37,7 +37,7 @@ class SdsPair {
class StringMap : public DenseSet {
public:
StringMap(std::pmr::memory_resource* res = std::pmr::get_default_resource()) : DenseSet(res) {
StringMap(MemoryResource* res = PMR_NS::get_default_resource()) : DenseSet(res) {
}
~StringMap();

View file

@ -20,7 +20,7 @@ namespace dfly {
class StringSet : public DenseSet {
public:
StringSet(std::pmr::memory_resource* res = std::pmr::get_default_resource()) : DenseSet(res) {
StringSet(MemoryResource* res = PMR_NS::get_default_resource()) : DenseSet(res) {
}
~StringSet();

View file

@ -32,7 +32,7 @@ namespace dfly {
using namespace std;
using absl::StrCat;
class DenseSetAllocator : public pmr::memory_resource {
class DenseSetAllocator : public PMR_NS::memory_resource {
public:
bool all_freed() const {
return alloced_ == 0;
@ -40,17 +40,17 @@ class DenseSetAllocator : public pmr::memory_resource {
void* do_allocate(size_t bytes, size_t alignment) override {
alloced_ += bytes;
void* p = pmr::new_delete_resource()->allocate(bytes, alignment);
void* p = PMR_NS::new_delete_resource()->allocate(bytes, alignment);
return p;
}
void do_deallocate(void* p, size_t bytes, size_t alignment) override {
alloced_ -= bytes;
return pmr::new_delete_resource()->deallocate(p, bytes, alignment);
return PMR_NS::new_delete_resource()->deallocate(p, bytes, alignment);
}
bool do_is_equal(const pmr::memory_resource& other) const noexcept override {
return pmr::new_delete_resource()->is_equal(other);
bool do_is_equal(const PMR_NS::memory_resource& other) const noexcept override {
return PMR_NS::new_delete_resource()->is_equal(other);
}
private:

View file

@ -300,7 +300,7 @@ void Connection::HandleRequests() {
if (absl::GetFlag(FLAGS_tcp_nodelay)) {
int val = 1;
CHECK_EQ(0, setsockopt(lsb->native_handle(), SOL_TCP, TCP_NODELAY, &val, sizeof(val)));
CHECK_EQ(0, setsockopt(lsb->native_handle(), IPPROTO_TCP, TCP_NODELAY, &val, sizeof(val)));
}
auto remote_ep = lsb->RemoteEndpoint();

View file

@ -71,7 +71,7 @@ class EngineShard {
return db_slice_;
}
std::pmr::memory_resource* memory_resource() {
PMR_NS::memory_resource* memory_resource() {
return &mi_resource_;
}

View file

@ -42,7 +42,7 @@ SlotStats& SlotStats::operator+=(const SlotStats& o) {
return *this;
}
DbTable::DbTable(std::pmr::memory_resource* mr)
DbTable::DbTable(PMR_NS::memory_resource* mr)
: prime(kInitSegmentLog, detail::PrimeTablePolicy{}, mr),
expire(0, detail::ExpireTablePolicy{}, mr), mcflag(0, detail::ExpireTablePolicy{}, mr),
top_keys({.enabled = absl::GetFlag(FLAGS_enable_top_keys_tracking)}) {

View file

@ -84,7 +84,7 @@ struct DbTable : boost::intrusive_ref_counter<DbTable, boost::thread_unsafe_coun
TopKeys top_keys;
explicit DbTable(std::pmr::memory_resource* mr);
explicit DbTable(PMR_NS::memory_resource* mr);
~DbTable();
void Clear();