From 02fff36e3e292baf0dc1b4d10680d2830efe2e4f Mon Sep 17 00:00:00 2001 From: Roman Gershman Date: Tue, 12 Sep 2023 10:42:06 +0300 Subject: [PATCH] Add build_rpm script and rpm spec (#1831) Also, link stdlib++ and libgcc statically. Signed-off-by: Roman Gershman --- .devcontainer/fedora/devcontainer.json | 14 ++++++ .github/actions/regression-tests/action.yml | 2 +- .github/workflows/release.yml | 39 ++++++++++++--- Makefile | 20 ++++++-- tests/README.md | 2 +- tools/packaging/Dockerfile.alpine-dev | 4 +- tools/packaging/Dockerfile.alpine-prod.wip | 2 +- tools/packaging/Dockerfile.ubuntu-dev | 4 +- tools/packaging/Dockerfile.ubuntu-prod | 2 +- tools/packaging/rpm/build_rpm.sh | 25 ++++++++++ tools/packaging/rpm/dragonfly.conf | 1 + tools/packaging/rpm/dragonfly.service | 1 + tools/packaging/rpm/dragonfly.spec | 53 +++++++++++++++++++++ 13 files changed, 149 insertions(+), 20 deletions(-) create mode 100644 .devcontainer/fedora/devcontainer.json create mode 100755 tools/packaging/rpm/build_rpm.sh create mode 120000 tools/packaging/rpm/dragonfly.conf create mode 120000 tools/packaging/rpm/dragonfly.service create mode 100644 tools/packaging/rpm/dragonfly.spec diff --git a/.devcontainer/fedora/devcontainer.json b/.devcontainer/fedora/devcontainer.json new file mode 100644 index 000000000..77a7caf38 --- /dev/null +++ b/.devcontainer/fedora/devcontainer.json @@ -0,0 +1,14 @@ +{ + "name": "helio", + "image": "ghcr.io/romange/fedora:30", + "customizations": { + "vscode": { + "extensions": [ + "ms-vscode.cpptools", + "ms-vscode.cmake-tools", + "ms-vscode.cpptools-themes", + "twxs.cmake" + ] + } + } +} diff --git a/.github/actions/regression-tests/action.yml b/.github/actions/regression-tests/action.yml index cce9d63e5..6bd72f00a 100644 --- a/.github/actions/regression-tests/action.yml +++ b/.github/actions/regression-tests/action.yml @@ -29,7 +29,7 @@ runs: ls -l ${GITHUB_WORKSPACE}/ cd ${GITHUB_WORKSPACE}/tests echo "Current commit is ${{github.sha}}" - pip install -r dragonfly/requirements.txt + pip3 install -r dragonfly/requirements.txt # used by PyTests export DRAGONFLY_PATH="${GITHUB_WORKSPACE}/${{inputs.build-folder-name}}/${{inputs.dfly-executable}}" export UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1 # to crash on errors diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 73a08ae6c..8641c2de2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -54,13 +54,14 @@ jobs: # Mount the artifacts directory as /artifacts in the container dockerRunArgs: | --volume "${{ github.workspace }}:/src" + # The shell to run commands with in the container shell: /bin/bash install: | export DEBIAN_FRONTEND=noninteractive apt update && apt install -q -y autoconf-archive cmake curl git libssl-dev \ - libunwind-dev ninja-build libtool gcc-9 g++-9 libboost-fiber-dev \ - libxml2-dev zip libzstd-dev debhelper moreutils bison + libunwind-dev ninja-build libtool gcc-9 g++-9 libboost-context-dev \ + zip libzstd-dev debhelper moreutils bison zlib1g-dev update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 40 \ --slave /usr/bin/g++ g++ /usr/bin/g++-9 run: | @@ -96,15 +97,25 @@ jobs: build-native: runs-on: ubuntu-latest needs: create-release + strategy: + matrix: + include: + # Build with these flags + - name: debian + container: ubuntu-dev:20 + - name: rpm + container: fedora:30 container: - image: ghcr.io/romange/ubuntu-dev:20 + image: ghcr.io/romange/${{ matrix.container }} steps: - uses: actions/checkout@v3 with: submodules: true - name: Configure run: | - apt update && apt install -y debhelper moreutils pip jq + if [ -f /etc/redhat-release ]; then + dnf install -y rpm-build libstdc++-static + fi - name: Build artifacts run: | # Work around https://github.com/actions/checkout/issues/766 @@ -112,8 +123,15 @@ jobs: git describe --always --tags ${{ github.sha }} ./tools/release.sh # once the build is over, we want to generate a Debian package - ./tools/packaging/generate_debian_package.sh build-opt/dragonfly-x86_64 + if [ -f /etc/debian_version ]; then + ./tools/packaging/generate_debian_package.sh build-opt/dragonfly-x86_64 + else + echo "TODO: build rpm" + fi + - name: Run regression tests + # Fedora 30 has older python packages, so this fails during requirements installation. + if : matrix.container != 'fedora:30' uses: ./.github/actions/regression-tests with: dfly-executable: dragonfly-x86_64 @@ -121,12 +139,19 @@ jobs: run-only-on-ubuntu-latest: false build-folder-name: build-opt - name: Save artifacts + if : matrix.container != 'fedora:30' # TODO run: | # place all artifacts at the same location mkdir -p results-artifacts - mv build-opt/dragonfly-*tar.gz results-artifacts - mv dragonfly_*.deb results-artifacts + if [ -f /etc/debian_version ]; then + mv build-opt/dragonfly-*tar.gz results-artifacts + mv dragonfly_*.deb results-artifacts + else + echo "TODO: upload rpm" + ls *.rpm + fi - name: Upload + if : matrix.container != 'fedora:30' # TODO uses: actions/upload-artifact@v3 with: name: dragonfly-amd64 diff --git a/Makefile b/Makefile index dff72b955..33999218d 100644 --- a/Makefile +++ b/Makefile @@ -2,18 +2,30 @@ BUILD_ARCH := $(shell uname -m) RELEASE_NAME := "dragonfly-${BUILD_ARCH}" HELIO_RELEASE := $(if $(HELIO_RELEASE),y,) -HELIO_RELEASE_FLAGS = -DHELIO_RELEASE_FLAGS="-flto -g1 -gz" + +HELIO_RELEASE_FLAGS = -DHELIO_RELEASE_FLAGS="-g1 -gz" HELIO_USE_STATIC_LIBS = ON HELIO_OPENSSL_USE_STATIC_LIBS = ON HELIO_ENABLE_GIT_VERSION = ON HELIO_WITH_UNWIND = OFF +# Some distributions (old fedora) have incorrect dependencies for crypto +# so we add -lz for them. +LINKER_FLAGS=-lz + # equivalent to: if $(uname_m) == x86_64 || $(uname_m) == amd64 ifneq (, $(filter $(BUILD_ARCH),x86_64 amd64)) HELIO_MARCH_OPT := -march=core2 -msse4.1 -mpopcnt -mtune=skylake endif +# For release builds we link statically libstdc++ and libgcc. Currently, +# all the release builds are performed by gcc. +ifeq ($(HELIO_RELEASE),y) + LINKER_FLAGS += -static-libstdc++ -static-libgcc +endif + HELIO_FLAGS = $(if $(HELIO_RELEASE),-release $(HELIO_RELEASE_FLAGS),) \ + -DCMAKE_EXE_LINKER_FLAGS="$(LINKER_FLAGS)" \ -DBoost_USE_STATIC_LIBS=$(HELIO_USE_STATIC_LIBS) \ -DOPENSSL_USE_STATIC_LIBS=$(HELIO_OPENSSL_USE_STATIC_LIBS) \ -DENABLE_GIT_VERSION=$(HELIO_ENABLE_GIT_VERSION) \ @@ -26,13 +38,11 @@ configure: build: cd build-opt; \ - ninja dragonfly; \ - ldd dragonfly + ninja dragonfly && ldd dragonfly build-debug: cd build-dbg; \ - ninja dragonfly; \ - ldd dragonfly + ninja dragonfly && ldd dragonfly package: cd build-opt; \ diff --git a/tests/README.md b/tests/README.md index 9bf5eb7b3..f6177195a 100644 --- a/tests/README.md +++ b/tests/README.md @@ -38,7 +38,7 @@ source /bin/activate ``` Then install all the required dependencies for the tests: ``` -pip install -r dragonfly/requirements.txt +pip3 install -r dragonfly/requirements.txt ``` ### Running the tests diff --git a/tools/packaging/Dockerfile.alpine-dev b/tools/packaging/Dockerfile.alpine-dev index 4ac3db97f..24d2af9d1 100644 --- a/tools/packaging/Dockerfile.alpine-dev +++ b/tools/packaging/Dockerfile.alpine-dev @@ -5,7 +5,7 @@ FROM alpine:3 as builder # "openssl-libs-static" fixes "Could NOT find OpenSSL, try to set the path to OpenSSL root folder in the" RUN apk add autoconf-archive automake bash bison boost-dev cmake coreutils \ - curl ccache git gcc gdb g++ libunwind-dev libtool libxml2-dev make ninja \ + curl ccache git gcc gdb g++ libunwind-dev libtool make ninja \ openssl-dev openssl-libs-static patch zip zstd-static # This is required to make static linking work @@ -30,7 +30,7 @@ COPY tools/docker/healthcheck.sh /usr/local/bin/healthcheck.sh COPY --from=builder /build/build-opt/dragonfly /usr/local/bin/ RUN apk --no-cache add libgcc libstdc++ \ - su-exec netcat-openbsd libxml2 icu boost-context && ldd /usr/local/bin/dragonfly + su-exec netcat-openbsd boost-context && ldd /usr/local/bin/dragonfly RUN addgroup -S -g 1000 dfly && adduser -S -G dfly -u 999 dfly RUN mkdir /data && chown dfly:dfly /data diff --git a/tools/packaging/Dockerfile.alpine-prod.wip b/tools/packaging/Dockerfile.alpine-prod.wip index babe3bfe0..43508b1ba 100644 --- a/tools/packaging/Dockerfile.alpine-prod.wip +++ b/tools/packaging/Dockerfile.alpine-prod.wip @@ -14,7 +14,7 @@ RUN ninja dragonfly FROM alpine:latest RUN addgroup -S -g 1000 dfly && adduser -S -G dfly -u 999 dfly -RUN apk --no-cache add libgcc libstdc++ libunwind boost1.77-fiber libxml2 \ +RUN apk --no-cache add libgcc libstdc++ libunwind boost-context \ 'su-exec>=0.2' netcat-openbsd openssl RUN mkdir /data && chown dfly:dfly /data diff --git a/tools/packaging/Dockerfile.ubuntu-dev b/tools/packaging/Dockerfile.ubuntu-dev index 8ff5c7ab4..5ca33c996 100644 --- a/tools/packaging/Dockerfile.ubuntu-dev +++ b/tools/packaging/Dockerfile.ubuntu-dev @@ -11,8 +11,8 @@ RUN \ export DEBIAN_FRONTEND=noninteractive && \ apt update && \ apt install -q -y autoconf-archive cmake curl git libssl-dev \ - libunwind-dev ninja-build libtool gcc-9 g++-9 libboost-fiber-dev \ - libxml2-dev zip libzstd-dev bison + libunwind-dev ninja-build libtool gcc-9 g++-9 libboost-context-dev \ + zip libzstd-dev bison RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 40 \ --slave /usr/bin/g++ g++ /usr/bin/g++-9 diff --git a/tools/packaging/Dockerfile.ubuntu-prod b/tools/packaging/Dockerfile.ubuntu-prod index 7d6664123..785123d49 100644 --- a/tools/packaging/Dockerfile.ubuntu-prod +++ b/tools/packaging/Dockerfile.ubuntu-prod @@ -20,7 +20,7 @@ FROM ubuntu:20.04 ARG QEMU_CPU ARG DEBIAN_FRONTEND=noninteractive -RUN apt clean && apt update && apt -y install netcat-openbsd ca-certificates redis-tools libxml2 +RUN apt clean && apt update && apt -y install netcat-openbsd ca-certificates redis-tools RUN groupadd -r -g 999 dfly && useradd -r -g dfly -u 999 dfly RUN mkdir /data && chown dfly:dfly /data diff --git a/tools/packaging/rpm/build_rpm.sh b/tools/packaging/rpm/build_rpm.sh new file mode 100755 index 000000000..c6972b650 --- /dev/null +++ b/tools/packaging/rpm/build_rpm.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +set -e + +# Get the full path of the binary +ARCHIVE=$(realpath "$1") +VERSION="$2" +echo "Preparing $ARCHIVE" + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" + +# Setup RPM build environment in a unique subdirectory under /tmp +RPM_ROOT=$(mktemp -d /tmp/rpmbuild_XXXXXX) +echo "Working dir is $RPM_ROOT" +mkdir -p $RPM_ROOT/{BUILD,RPMS,SOURCES,SPECS} + +# Put the archive and configuration files to the SOURCES directory +ln -s "$ARCHIVE" -t "$RPM_ROOT/SOURCES/" +cp $SCRIPT_DIR/dragonfly.service $RPM_ROOT/SOURCES/ +cp $SCRIPT_DIR/dragonfly.conf $RPM_ROOT/SOURCES/ + +cp $SCRIPT_DIR/dragonfly.spec $RPM_ROOT/SPECS/ + +rpmbuild --define "_topdir $RPM_ROOT" --define "version $VERSION" -bb "$RPM_ROOT/SPECS/dragonfly.spec" +mv $RPM_ROOT/RPMS/*.rpm ./ diff --git a/tools/packaging/rpm/dragonfly.conf b/tools/packaging/rpm/dragonfly.conf new file mode 120000 index 000000000..2ddf8ce10 --- /dev/null +++ b/tools/packaging/rpm/dragonfly.conf @@ -0,0 +1 @@ +../debian/dragonfly.conf \ No newline at end of file diff --git a/tools/packaging/rpm/dragonfly.service b/tools/packaging/rpm/dragonfly.service new file mode 120000 index 000000000..72d047816 --- /dev/null +++ b/tools/packaging/rpm/dragonfly.service @@ -0,0 +1 @@ +../debian/dragonfly.service \ No newline at end of file diff --git a/tools/packaging/rpm/dragonfly.spec b/tools/packaging/rpm/dragonfly.spec new file mode 100644 index 000000000..6b6229ee3 --- /dev/null +++ b/tools/packaging/rpm/dragonfly.spec @@ -0,0 +1,53 @@ +%define pkg_name dragonfly +%define archive dragonfly-%{_arch}.tar.gz + +# How the package name looks like +%define _build_name_fmt %%{NAME}.%%{ARCH}.rpm + +Name: %{pkg_name} +Version: %{version} +Release: 1%{?dist} +Summary: DragonflyDB memory store +License: BUSL-1.1 +URL: https://www.dragonflydb.io +Source0: %{archive} +Source1: dragonfly.service +Source2: dragonfly.conf +Group: Applications/System +Provides: user(dfly) +Provides: group(dfly) + +%description +DragonflyDB is a vertically scalable and memory efficient in-memory store +that is compatible with Redis OSS and Memcached. + +%pre + +getent group dfly >/dev/null || groupadd -r dfly +getent passwd dfly >/dev/null || useradd -r -g dfly -M -s /sbin/nologin -c "User for DragonflyDB service" dfly + +%prep + +%build +tar xvfz %{SOURCE0} +mv ./dragonfly-%{_arch} ./dragonfly + +%install +mkdir -p %{buildroot}/usr/local/bin +mkdir -p %{buildroot}/etc/dragonfly + +install -m 755 ./dragonfly %{buildroot}/usr/local/bin/ +mkdir -p %{buildroot}/usr/lib/systemd/system +cp %{SOURCE1} %{buildroot}/usr/lib/systemd/system/ +cp %{SOURCE2} %{buildroot}/etc/dragonfly/ + +%clean +rm -rf %{buildroot} +rm -rf %{_builddir}/* + +%files +%attr(-,dfly,dfly) /usr/local/bin/dragonfly +%attr(-,dfly,dfly) /usr/lib/systemd/system/dragonfly.service +%attr(-,dfly,dfly) /etc/dragonfly/dragonfly.conf + +%changelog