diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 8c0f9d22c..ea7062931 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -24,10 +24,39 @@ env: IS_PRERELEASE: ${{ github.event.release.prerelease || github.event.inputs.PRERELEASE }} jobs: -# =============================================================== -# Building Dev Images -# =============================================================== + # This workaround is needed, as it's not possible to reference env.FOOBAR directly at this stage + # - for ex. https://github.com/actions/runner/issues/1189 + release-container-prereq: + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.release-param.outputs.tag }} + tag_latest: ${{ steps.release-param.outputs.tag_latest }} + steps: + - id: release-param + run: | + # If env.IS_PRERELEASE is true, set tag to alpha and do not enable tag_latest + # If env.IS_PRERELEASE is not true (aka false), don't set an extra tag and enable tag_latest + echo "::set-output name=tag::${{ env.IS_PRERELEASE == 'true' && 'alpha' }}" + echo "::set-output name=tag_latest::${{ env.IS_PRERELEASE == 'true' && 'false' || 'true' }}" + + release-container: + needs: release-container-prereq + uses: ./.github/workflows/reusable-container-workflow.yaml + with: + build_type: prod + tag: ${{ needs.release-container-prereq.outputs.tag }} + tag_latest: ${{ needs.release-container-prereq.outputs.tag_latest == 'true' }} + image: ghcr.io/${{ github.repository }} + registry: ghcr.io + registry_username: ${{ github.repository_owner }} + fetch_release: true + release_version: ${{ github.event.inputs.TAG_NAME || github.event.release.tag_name }} + secrets: + registry_password: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + release: + needs: [release-container-prereq, release-container] runs-on: ubuntu-latest steps: - name: print_env @@ -41,76 +70,6 @@ jobs: - name: Install helm uses: azure/setup-helm@v3 - - uses: dsaltares/fetch-gh-release-asset@master - with: - version: 'tags/${{ env.TAG_NAME }}' - regex: true - file: "dragonfly-.*\\.tar\\.gz" - target: 'releases/' - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up QEMU - id: qemu - uses: docker/setup-qemu-action@v1 - with: - platforms: arm64,amd64 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract artifacts - run: | - echo "Event prerelease ${{ github.event.release.prerelease }}" - echo "Input prerelease ${{ github.event.inputs.PRERELEASE }}" - ls -l - ls -l releases - for f in releases/*.tar.gz; do tar xvfz $f -C releases; done - rm releases/*.tar.gz - - - name: Build release image - if: env.IS_PRERELEASE != 'true' - uses: docker/build-push-action@v3 - with: - context: . - platforms: linux/amd64,linux/arm64 - - # Define QEMU settings inside the builder - build-args: | - QEMU_CPU=max,pauth-impdef=on - - push: ${{ github.event_name != 'pull_request' }} - tags: | - ghcr.io/${{ github.repository }}:ubuntu - ghcr.io/${{ github.repository }}:latest - ghcr.io/${{ github.repository }}:${{ env.TAG_NAME }} - ghcr.io/${{ github.repository }}:${{ env.TAG_NAME }}-ubuntu - - file: tools/docker/Dockerfile.ubuntu-prod - cache-from: type=registry,ref=${{ github.repository }}:latest - cache-to: type=inline - - - name: Build pre-release image - if: env.IS_PRERELEASE == 'true' - uses: docker/build-push-action@v3 - with: - context: . - platforms: linux/amd64,linux/arm64 - - push: ${{ github.event_name != 'pull_request' }} - tags: | - ghcr.io/${{ github.repository }}:alpha - ghcr.io/${{ github.repository }}:${{ env.TAG_NAME }} - file: tools/docker/Dockerfile.ubuntu-prod - cache-from: type=registry,ref=${{ github.repository }}:latest - cache-to: type=inline - - name: Configure Git if: env.IS_PRERELEASE != 'true' run: | diff --git a/.github/workflows/docker-weekly.yml b/.github/workflows/docker-weekly.yml new file mode 100644 index 000000000..bc54f18e8 --- /dev/null +++ b/.github/workflows/docker-weekly.yml @@ -0,0 +1,24 @@ +name: weekly docker build + +on: + schedule: + # Monday midnight + - cron: '0 0 * * 1' + workflow_dispatch: + +permissions: + packages: write + contents: write + id-token: write + +jobs: + weekly-container-build: + uses: ./.github/workflows/reusable-container-workflow.yaml + with: + build_type: dev + tag: alpha + image: ghcr.io/${{ github.repository }} + registry: ghcr.io + registry_username: ${{ github.repository_owner }} + secrets: + registry_password: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/reusable-container-workflow.yaml b/.github/workflows/reusable-container-workflow.yaml new file mode 100644 index 000000000..cfa0a7a7b --- /dev/null +++ b/.github/workflows/reusable-container-workflow.yaml @@ -0,0 +1,146 @@ +name: Reusable Container Build Workflow + +on: + workflow_call: + inputs: + # Which suffix to look for with the Dockerfile. Can be dev or prod + build_type: + required: true + type: string + # For example 'alpha', for pre-release or weekly builds + tag: + required: false + type: string + # Is this a final release? Then we set this to true, so the 'latest' tag is updated + tag_latest: + required: false + type: boolean + # The container image dragonflydb/dragonfly + image: + required: true + type: string + # ghcr.io / hub.docker.com / quay.io / you name it + registry: + required: true + type: string + # Username used to login to the registry + registry_username: + required: true + type: string + # Do we have to fetch release assets? Then set this to true. + # Not required for build_type == dev, as they entirely build from source + # But for build_type == prod, as they're based on the release assets + fetch_release: + required: false + type: boolean + # Which version are we fetching? Should be identical to the release version. + # For example v0.12.0 + release_version: + required: false + type: string + secrets: + # Password used to login to the registry + registry_password: + required: true + # Github Personal Access Token used to fetch assets from a release + GH_TOKEN: + required: false + +jobs: + container-build: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - flavor: alpine + dockerfile: tools/docker/Dockerfile.alpine + tag_main: false + - flavor: ubuntu + dockerfile: tools/docker/Dockerfile.ubuntu + tag_main: true + steps: + - name: checkout + uses: actions/checkout@v3 + with: + fetch-depth: 1 + submodules: true + + - if: inputs.fetch_release + name: Fetch release asset + uses: dsaltares/fetch-gh-release-asset@1.1.0 + with: + version: "tags/${{ inputs.release_version }}" + regex: true + file: "dragonfly-.*\\.tar\\.gz" + target: 'releases/' + token: ${{ secrets.GH_TOKEN }} + + - if: inputs.fetch_release + name: Extract artifacts + run: | + echo "Event prerelease ${{ github.event.release.prerelease }}" + echo "Input prerelease ${{ github.event.inputs.PRERELEASE }}" + ls -l + ls -l releases + for f in releases/*.tar.gz; do tar xvfz $f -C releases; done + rm releases/*.tar.gz + + - name: Set up QEMU + id: qemu + uses: docker/setup-qemu-action@v1 + with: + platforms: arm64,amd64 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ inputs.registry }} + username: ${{ inputs.registry_username }} + password: ${{ secrets.registry_password }} + + - name: Docker meta + id: metadata + uses: docker/metadata-action@v3 + with: + images: | + ${{ inputs.image }} + tags: | + # will set tag 'latest' for ubuntu build on production push + # inputs.tag_latest will be true on when triggered by docker-release.yml + # matrix.tag_main will only be true for ubuntu flavor + type=raw,value=latest,enable=${{ matrix.tag_main && inputs.tag_latest }} + + # set a tag like 'alpine' or 'ubuntu', if we're setting 'latest' during this build as well + type=raw,value=${{ matrix.flavor }},enable=${{ inputs.tag_latest }} + + # will set tag like 'alpha' for ubuntu build, if inputs.tag is not empty + # but will set the non-flavored tag only, if matrix.tag_main is true + type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != 'false' && matrix.tag_main }} + + # will set tag like 'alpha-(ubuntu|alpine)', if inputs.tag is not empty + type=raw,value=${{ inputs.tag }}-${{ matrix.flavor }},enable=${{ inputs.tag != 'false' }} + + # will set tag like 'v0.12.0' for ubuntu build, if inputs.release_version is not empty + # but will set the non-flavored tag only, if matrix.tag_main is true + type=raw,value=${{ inputs.release_version }},enable=${{ matrix.tag_main && inputs.release_version != '' }} + + # will set tag like 'v0.12.0-(ubuntu|alpine)', if inputs.release_version is not empty + type=raw,value=${{ inputs.release_version }}-${{ matrix.flavor }},enable=${{ inputs.release_version != '' }} + + - if: ${{ hashFiles(format('{0}-{1}', matrix.dockerfile, inputs.build_type)) }} + name: Build release image + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/amd64,linux/arm64 + build-args: | + QEMU_CPU=max,pauth-impdef=on + push: true + tags: | + ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} + file: ${{ matrix.dockerfile }}-${{ inputs.build_type }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..dd8521345 --- /dev/null +++ b/Makefile @@ -0,0 +1,43 @@ +BUILD_ARCH := $(shell uname -m) +RELEASE_NAME := "dragonfly-${BUILD_ARCH}" + +HELIO_RELEASE := $(if $(HELIO_RELEASE),y,) +HELIO_RELEASE_FLAGS = -DHELIO_RELEASE_FLAGS="-flto" +HELIO_USE_STATIC_LIBS = ON +HELIO_OPENSSL_USE_STATIC_LIBS = ON +HELIO_ENABLE_GIT_VERSION = ON +HELIO_WITH_UNWIND = OFF + +HELIO_FLAGS = $(if $(HELIO_RELEASE),-release $(HELIO_RELEASE_FLAGS),) \ + -DBoost_USE_STATIC_LIBS=$(HELIO_USE_STATIC_LIBS) \ + -DOPENSSL_USE_STATIC_LIBS=$(HELIO_OPENSSL_USE_STATIC_LIBS) \ + -DENABLE_GIT_VERSION=$(HELIO_ENABLE_GIT_VERSION) \ + -DWITH_UNWIND=$(HELIO_WITH_UNWIND) \ + +.PHONY: default + +configure: + ./helio/blaze.sh $(HELIO_FLAGS) + +build: + cd build-opt; \ + ninja dragonfly; \ + ldd dragonfly + +build-debug: + cd build-dbg; \ + ninja dragonfly; \ + ldd dragonfly + +package: + cd build-opt; \ + mv dragonfly $(RELEASE_NAME); \ + tar cvfz $(RELEASE_NAME).unstripped.tar.gz $(RELEASE_NAME) ../LICENSE.md; \ + strip $(RELEASE_NAME); \ + tar cvfz $(RELEASE_NAME).tar.gz $(RELEASE_NAME) ../LICENSE.md + +release: configure build + +release-debug: configure build-debug + +default: release diff --git a/tools/docker/Dockerfile.alpine-dev b/tools/docker/Dockerfile.alpine-dev new file mode 100644 index 000000000..392ddb37c --- /dev/null +++ b/tools/docker/Dockerfile.alpine-dev @@ -0,0 +1,44 @@ +# syntax=docker/dockerfile:1 +FROM gcr.io/cadvisor/cadvisor:v0.46.0 as libpfm_donor + +FROM alpine:3.17.0 as builder + +# "openssl-libs-static" fixes "Could NOT find OpenSSL, try to set the path to OpenSSL root folder in the" +RUN apk add autoconf-archive automake bash bison boost-dev cmake coreutils \ + curl ccache git gcc gdb g++ libunwind-dev libtool libxml2-dev make ninja \ + openssl-dev openssl-libs-static patch zip zstd-dev + +# This is required to make static linking work +RUN ls -1 /usr/lib/libboost_*.so | while read -r _file; do ln -sfv ${_file} ${_file//.so/.a}; done + +# Borrow libpfm from cadvisor, so we don't have to build it ourselves +# https://github.com/google/cadvisor/blob/master/deploy/Dockerfile +COPY --from=libpfm_donor /usr/local/lib/libpfm.so* /usr/local/lib/ + +WORKDIR /build + +COPY . ./ + +RUN make HELIO_RELEASE=y release + +RUN build-opt/dragonfly --version + +FROM alpine:3.17.0 + +RUN apk --no-cache add libgcc libstdc++ libunwind boost1.80-fiber zstd-dev su-exec netcat-openbsd + +RUN addgroup -S -g 1000 dfly && adduser -S -G dfly -u 999 dfly +RUN mkdir /data && chown dfly:dfly /data + +VOLUME /data +WORKDIR /data +COPY tools/docker/entrypoint.sh /usr/local/bin/entrypoint.sh +COPY tools/docker/healthcheck.sh /usr/local/bin/healthcheck.sh +COPY --from=builder /build/build-opt/dragonfly /usr/local/bin/ + +HEALTHCHECK CMD /usr/local/bin/healthcheck.sh +ENTRYPOINT ["entrypoint.sh"] + +EXPOSE 6379 + +CMD ["dragonfly", "--logtostderr"] diff --git a/tools/docker/Dockerfile.alpine-prod b/tools/docker/Dockerfile.alpine-prod.wip similarity index 85% rename from tools/docker/Dockerfile.alpine-prod rename to tools/docker/Dockerfile.alpine-prod.wip index 144f81ae0..b7e62ca74 100644 --- a/tools/docker/Dockerfile.alpine-prod +++ b/tools/docker/Dockerfile.alpine-prod.wip @@ -12,9 +12,6 @@ WORKDIR build-opt RUN ninja dragonfly FROM alpine:latest -ARG ORG_NAME=dragonflydb -LABEL org.opencontainers.image.title Dragonfly -LABEL org.opencontainers.image.source https://github.com/${ORG_NAME}/dragonfly RUN addgroup -S -g 1000 dfly && adduser -S -G dfly -u 999 dfly RUN apk --no-cache add libgcc libstdc++ libunwind boost1.77-fiber \ diff --git a/tools/docker/Dockerfile.ubuntu-dev b/tools/docker/Dockerfile.ubuntu-dev new file mode 100644 index 000000000..9681974c6 --- /dev/null +++ b/tools/docker/Dockerfile.ubuntu-dev @@ -0,0 +1,56 @@ +# syntax=docker/dockerfile:1 +FROM ubuntu:20.04 as builder + +RUN \ + rm -f /etc/apt/apt.conf.d/docker-clean; \ + echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache + +RUN \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + export DEBIAN_FRONTEND=noninteractive && \ + apt update && \ + apt install -q -y autoconf-archive cmake curl git libssl-dev \ + libunwind-dev ninja-build libtool gcc-9 g++-9 libboost-fiber-dev \ + libxml2-dev zip libzstd-dev + +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 40 \ + --slave /usr/bin/g++ g++ /usr/bin/g++-9 + +WORKDIR /build + +COPY . ./ + +RUN make HELIO_RELEASE=y release + +RUN build-opt/dragonfly --version + +RUN curl -O https://raw.githubusercontent.com/ncopa/su-exec/212b75144bbc06722fbd7661f651390dc47a43d1/su-exec.c && \ + gcc -Wall -O2 su-exec.c -o su-exec + +FROM ubuntu:20.04 + +RUN \ + --mount=type=tmpfs,target=/var/cache/apt \ + --mount=type=tmpfs,target=/var/lib/apt \ + export DEBIAN_FRONTEND=noninteractive && \ + apt update && \ + apt install -q -y netcat-openbsd + +RUN groupadd -r -g 999 dfly && useradd -r -g dfly -u 999 dfly +RUN mkdir /data && chown dfly:dfly /data + +VOLUME /data +WORKDIR /data +COPY tools/docker/entrypoint.sh /usr/local/bin/entrypoint.sh +COPY tools/docker/healthcheck.sh /usr/local/bin/healthcheck.sh +COPY --from=builder /build/su-exec /usr/local/bin/ +COPY --from=builder /build/build-opt/dragonfly /usr/local/bin/ + +HEALTHCHECK CMD /usr/local/bin/healthcheck.sh +ENTRYPOINT ["entrypoint.sh"] + +# For inter-container communication. +EXPOSE 6379 + +CMD ["dragonfly", "--logtostderr"] diff --git a/tools/docker/Dockerfile.ubuntu-prod b/tools/docker/Dockerfile.ubuntu-prod index 0b64708b2..2f7557e16 100644 --- a/tools/docker/Dockerfile.ubuntu-prod +++ b/tools/docker/Dockerfile.ubuntu-prod @@ -19,12 +19,8 @@ FROM ubuntu:20.04 # ARG in fact change the env vars during the build process # ENV persist the env vars for the built image as well. ARG QEMU_CPU -ARG ORG_NAME=dragonflydb ARG DEBIAN_FRONTEND=noninteractive -LABEL org.opencontainers.image.title Dragonfly -LABEL org.opencontainers.image.source https://github.com/${ORG_NAME}/dragonfly - RUN apt clean && apt update && apt -y install netcat-openbsd diff --git a/tools/release.sh b/tools/release.sh index 983846af8..e2909ad72 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -2,22 +2,18 @@ set -e -if ! [ -d helio ]; then - echo "could not find helio" +if ! [ -f "helio/blaze.sh" ]; then + echo "ERROR" + echo "Could not find helio. Please only run this script from repo root." + echo "If you are already on the repo root, you might've cloned without submodules." + echo "Try running 'git submodule update --init --recursive'" exit 1 fi -ARCH=`uname -m` -NAME="dragonfly-${ARCH}" - pwd -./helio/blaze.sh -release -DBoost_USE_STATIC_LIBS=ON -DOPENSSL_USE_STATIC_LIBS=ON \ - -DENABLE_GIT_VERSION=ON -DWITH_UNWIND=OFF -DHELIO_RELEASE_FLAGS="-flto" -cd build-opt -ninja dragonfly && ldd dragonfly -./dragonfly --version -mv dragonfly $NAME -tar cvfz $NAME.unstripped.tar.gz $NAME ../LICENSE.md -strip $NAME -tar cvfz $NAME.tar.gz $NAME ../LICENSE.md +make HELIO_RELEASE=y release + +build-opt/dragonfly --version + +make package