From e3eb8518fd09259f09ba8d97b1bab3540e9701fd Mon Sep 17 00:00:00 2001 From: adiholden Date: Wed, 17 Jul 2024 14:34:48 +0300 Subject: [PATCH] feat(test): Improve benchmark workflow (#3330) Signed-off-by: adi_holden --- .github/workflows/benchmark.yml | 47 ++++++++++++++++++++++++-- tools/benchmark/k8s-benchmark-job.yaml | 2 +- tools/benchmark/post_run_checks.py | 10 +++++- 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index e1b2b2d67..54f85e304 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -22,7 +22,7 @@ jobs: app.kubernetes.io/created-by: "dragonfly-operator" name: "dragonfly-sample" spec: - image: "ghcr.io/dragonflydb/dragonfly-weekly:latest" + image: "ghcr.io/dragonflydb/dragonfly:latest" args: ["--cache_mode"] replicas: 2 resources: @@ -89,7 +89,27 @@ jobs: kubectl wait -n ${{ steps.setup.outputs.namespace }} pods --selector app=dragonfly-sample --for condition=Ready --timeout=120s kubectl describe -n ${{ steps.setup.outputs.namespace }} pod dragonfly-sample-0 - - name: Run Benchmark + - name: Run Memtier Benchmark + shell: bash + run: | + kubectl apply -n ${{ steps.setup.outputs.namespace }} -f tools/benchmark/k8s-benchmark-job.yaml + + - name: Version upgrade + shell: bash + run: | + # benchmark is running, wait for 30 seconds before version upgrade + sleep 30 + kubectl patch dragonfly dragonfly-sample -n ${{ steps.setup.outputs.namespace }} --type merge -p '{"spec":{"image":"ghcr.io/dragonflydb/dragonfly-weekly:latest"}}' + + - name: Wait for Memtier Benchmark fail + shell: bash + run: | + # Memtier benchmark run will fail at some point because old master shutdown on version upgrade + kubectl wait --for=condition=failed --timeout=120s -n ${{ steps.setup.outputs.namespace }} jobs/memtier-benchmark 2>/dev/null + kubectl logs -n ${{ steps.setup.outputs.namespace }} -f jobs/memtier-benchmark + kubectl delete -n ${{ steps.setup.outputs.namespace }} jobs/memtier-benchmark + + - name: Run Memtier Benchmark again shell: bash run: | kubectl apply -n ${{ steps.setup.outputs.namespace }} -f tools/benchmark/k8s-benchmark-job.yaml @@ -116,7 +136,6 @@ jobs: - name: Server checks run: | nohup kubectl port-forward -n ${{ steps.setup.outputs.namespace }} service/dragonfly-sample 6379:6379 & - pip install -r tools/requirements.txt python3 tools/benchmark/post_run_checks.py @@ -138,6 +157,15 @@ jobs: command: | kubectl logs -n ${{ steps.setup.outputs.namespace }} dragonfly-sample-1 + - name: Describe dragonflydb object + uses: nick-fields/retry@v3 + if: always() + with: + timeout_minutes: 1 + max_attempts: 3 + command: | + kubectl describe dragonflies.dragonflydb.io -n ${{ steps.setup.outputs.namespace }} dragonfly-sample + - name: Scale down to zero if: always() run: | @@ -152,3 +180,16 @@ jobs: set -x kubectl delete namespace ${{ steps.setup.outputs.namespace }} kubectl delete namespace dragonfly-operator-system + + - name: Send notification on failure + if: failure() && github.ref == 'refs/heads/main' + shell: bash + run: | + job_link="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + message="Benchmark tests failed.\\n Job Link: ${job_link}\\n" + + curl -s \ + -X POST \ + -H 'Content-Type: application/json' \ + '${{ secrets.GSPACES_BOT_DF_BUILD }}' \ + -d '{"text": "'"${message}"'"}' diff --git a/tools/benchmark/k8s-benchmark-job.yaml b/tools/benchmark/k8s-benchmark-job.yaml index fababcdde..e96a0dbbc 100644 --- a/tools/benchmark/k8s-benchmark-job.yaml +++ b/tools/benchmark/k8s-benchmark-job.yaml @@ -11,7 +11,7 @@ spec: - name: memtier image: redislabs/memtier_benchmark:latest args: - - memtier_benchmark --pipeline=30 --key-maximum=10000 -c 10 -t 2 --requests=500000 --expiry-range=10-100 --reconnect-interval=10000 --distinct-client-seed --hide-histogram -s dragonfly-sample + - memtier_benchmark --pipeline=30 --key-maximum=100000 -c 10 -t 2 --test-time=600 --reconnect-interval=10000 --distinct-client-seed --hide-histogram -s dragonfly-sample command: - sh # This is important! without it memtier cannot DIG the dragonfly SVC domain - -c diff --git a/tools/benchmark/post_run_checks.py b/tools/benchmark/post_run_checks.py index a7a22ce7d..b0e3ad549 100644 --- a/tools/benchmark/post_run_checks.py +++ b/tools/benchmark/post_run_checks.py @@ -7,6 +7,10 @@ def main(): max_unaccounted = 200 * 1024 * 1024 # 200mb client = redis.Redis(decode_responses=True) + info = client.info("server") + # Check version upgrade finsihed from last released version to last weekly docker build + assert info["dragonfly_version"] == "df-HEAD-HASH-NOTFOUND" + info = client.info("memory") print(f'Used memory {info["used_memory"]}, rss {info["used_memory_rss"]}') assert info["used_memory_rss"] - info["used_memory"] < max_unaccounted @@ -26,7 +30,11 @@ def main(): time.sleep(1) replication_state = client.info("replication")["slave0"] - assert replication_state["lag"] == 0, f"Lag is bad, expected 0, got {replication_state['lag']}" + if replication_state["lag"] != 0: + print(f"Lag is bad, expected 0, got {replication_state['lag']}") + info = client.info("all") + print(f"Info all output: {info}") + assert False if __name__ == "__main__":