From 2c8cb230988f4f98c6b8a97bd68fe1fb5cee5559 Mon Sep 17 00:00:00 2001 From: Roman Gershman Date: Mon, 25 Apr 2022 12:06:31 +0300 Subject: [PATCH] Few improvements. 1. Docker build now builds for arm64 as well. 2. Add bind option to specify on which interface the server should listen. 3. Automatically deduce maxmemory setting. --- .github/workflows/ci.yml | 4 ++-- .github/workflows/docker-release.yml | 13 ++++++++++--- helio | 2 +- src/server/dfly_main.cc | 27 ++++++++++++++++++++++++--- src/server/main_service.cc | 5 +++-- tools/docker/Dockerfile.ubuntu-prod | 2 +- 6 files changed, 41 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5da84565..983106048 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,10 +20,10 @@ jobs: strategy: matrix: # Test of these containers - container: ["ubuntu-dev", "alpine-dev"] + container: ["ubuntu-dev:20", "alpine-dev:latest"] timeout-minutes: 30 container: - image: ghcr.io/${{ github.repository_owner }}/${{ matrix.container }}:latest + image: ghcr.io/${{ github.repository_owner }}/${{ matrix.container }} credentials: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 05ca9d648..ca2f348ef 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -15,6 +15,13 @@ jobs: with: submodules: true + - + name: Set up QEMU + id: qemu + uses: docker/setup-qemu-action@v1 + with: + platforms: arm64,amd64 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 @@ -29,7 +36,7 @@ jobs: uses: docker/build-push-action@v2 with: context: . - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 push: ${{ github.event_name != 'pull_request' }} tags: | ghcr.io/${{ github.actor }}/dragonfly-alpine:latest @@ -39,8 +46,8 @@ jobs: uses: docker/build-push-action@v2 with: context: . - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 push: ${{ github.event_name != 'pull_request' }} tags: | ghcr.io/${{ github.actor }}/dragonfly-ubuntu:latest - file: tools/docker/Dockerfile.ubuntu-prod \ No newline at end of file + file: tools/docker/Dockerfile.ubuntu-prod:20 \ No newline at end of file diff --git a/helio b/helio index 0ef31fbae..9b96ae52b 160000 --- a/helio +++ b/helio @@ -1 +1 @@ -Subproject commit 0ef31fbae12111fec2d49eab98922961ff871049 +Subproject commit 9b96ae52be8ccf35c959c366757ae30174d84a0e diff --git a/src/server/dfly_main.cc b/src/server/dfly_main.cc index 9fcdeb144..09c320d3c 100644 --- a/src/server/dfly_main.cc +++ b/src/server/dfly_main.cc @@ -6,9 +6,11 @@ #include #include "base/init.h" -#include "base/proc_util.h" +#include "base/proc_util.h" // for GetKernelVersion #include "facade/dragonfly_listener.h" +#include "io/proc_reader.h" #include "server/main_service.h" +#include "strings/human_readable.h" #include "util/accept_server.h" #include "util/uring/uring_pool.h" #include "util/varz.h" @@ -17,10 +19,14 @@ DECLARE_uint32(port); DECLARE_uint32(memcache_port); DECLARE_uint64(maxmemory); DEFINE_bool(use_large_pages, false, "If true - uses large memory pages for allocations"); +DEFINE_string(bind, "", + "Bind address. If empty - binds on all interfaces. " + "It's not advised due to security implications."); using namespace util; using namespace std; using namespace facade; +using strings::HumanReadableNumBytes; namespace dfly { @@ -33,7 +39,12 @@ bool RunEngine(ProactorPool* pool, AcceptServer* acceptor) { Service service(pool); service.Init(acceptor); - acceptor->AddListener(FLAGS_port, new Listener{Protocol::REDIS, &service}); + const char* bind_addr = FLAGS_bind.empty() ? nullptr : FLAGS_bind.c_str(); + + error_code ec = + acceptor->AddListener(bind_addr, FLAGS_port, new Listener{Protocol::REDIS, &service}); + + LOG_IF(FATAL, ec) << "Cound not open port " << FLAGS_port << ", error: " << ec.message(); if (FLAGS_memcache_port > 0) { acceptor->AddListener(FLAGS_memcache_port, new Listener{Protocol::MEMCACHE, &service}); @@ -69,13 +80,23 @@ int main(int argc, char* argv[]) { CHECK_LT(kver.minor, 99u); dfly::kernel_version = kver.major * 100 + kver.minor; + if (FLAGS_maxmemory == 0) { + LOG(INFO) << "maxmemory has not been specified. Deciding myself...."; + + io::Result res = io::ReadMemInfo(); + size_t available = res->mem_avail; + size_t maxmemory = size_t(0.8 * available); + LOG(INFO) << "Found " << HumanReadableNumBytes(available) + << " available memory. Setting maxmemory to " << HumanReadableNumBytes(maxmemory); + FLAGS_maxmemory = maxmemory; + } + if (FLAGS_use_large_pages) { mi_option_enable(mi_option_large_os_pages); } mi_option_enable(mi_option_show_errors); mi_option_set(mi_option_max_warnings, 0); - uring::UringPool pp{1024}; pp.Run(); diff --git a/src/server/main_service.cc b/src/server/main_service.cc index 1942049f7..f874f3a10 100644 --- a/src/server/main_service.cc +++ b/src/server/main_service.cc @@ -40,7 +40,9 @@ extern "C" { DEFINE_uint32(port, 6379, "Redis port"); DEFINE_uint32(memcache_port, 0, "Memcached port"); DECLARE_string(requirepass); -DEFINE_uint64(maxmemory, 0, "Limit on maximum-memory that is used by the database"); +DEFINE_uint64(maxmemory, 0, + "Limit on maximum-memory that is used by the database." + "0 - means the program will automatically determine its maximum memory usage"); namespace dfly { @@ -997,7 +999,6 @@ void Service::RegisterCommands() { server_family_.Register(®istry_); - if (VLOG_IS_ON(1)) { LOG(INFO) << "Multi-key commands are: "; registry_.Traverse([](std::string_view key, const CI& cid) { diff --git a/tools/docker/Dockerfile.ubuntu-prod b/tools/docker/Dockerfile.ubuntu-prod index 410aea624..4bfdc581f 100644 --- a/tools/docker/Dockerfile.ubuntu-prod +++ b/tools/docker/Dockerfile.ubuntu-prod @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -from ghcr.io/romange/ubuntu-dev as builder +from ghcr.io/romange/ubuntu-dev:20 as builder WORKDIR /build COPY src/ ./src/