#!/usr/bin/env bash
# ==============================================================================
# SharpMailserver Open Reproducible Benchmark Suite (v1.1.0)
# Published by iqmeta GmbH — https://sharpmailserver.com/trust/#benchmarks
#
# Measures:
# 1. Exact host hardware, kernel, CPU governor & server idle RSS (MB)
# 2. Concurrent ESMTP inbound throughput (msgs/sec, p50/p95/p99 latency)
# using a realistic enterprise payload mix (16 KB MIME + 45 KB EN 16931 XML)
# 3. Peak RSS under sustained parallel load & post-GC working set recovery
# 4. Authenticated HTTP/JMAP full-text & ZUGFeRD search p50/p95/p99 latency
#
# Usage:
# bash run-benchmark.sh \
# --smtp-host 127.0.0.1 --smtp-port 25 \
# --http-url http://127.0.0.1:8080 \
# --recipient bench@firma.de \
# --concurrency 16 --messages 2000
# ==============================================================================
set -euo pipefail
SMTP_HOST="127.0.0.1"
SMTP_PORT="25"
HTTP_URL="http://127.0.0.1:80"
RECIPIENT="admin@example.com"
ADMIN_USER=""
ADMIN_PASS=""
CONCURRENCY=16
TOTAL_MESSAGES=2000
SEARCH_ITERATIONS=200
SERVER_PID=""
while [ $# -gt 0 ]; do
case "$1" in
--smtp-host) SMTP_HOST="$2"; shift 2 ;;
--smtp-port) SMTP_PORT="$2"; shift 2 ;;
--http-url) HTTP_URL="$2"; shift 2 ;;
--recipient) RECIPIENT="$2"; shift 2 ;;
--user) ADMIN_USER="$2"; shift 2 ;;
--password) ADMIN_PASS="$2"; shift 2 ;;
--concurrency) CONCURRENCY="$2"; shift 2 ;;
--messages) TOTAL_MESSAGES="$2"; shift 2 ;;
--search-runs) SEARCH_ITERATIONS="$2"; shift 2 ;;
--pid) SERVER_PID="$2"; shift 2 ;;
*) echo "Unknown option: $1"; exit 1 ;;
esac
done
if ! command -v python3 >/dev/null 2>&1; then
echo "Error: python3 (standard library only) is required to drive high-precision async SMTP/HTTP timing." >&2
exit 1
fi
read_rss_mb() {
local pid="$1"
if [ -z "${pid}" ]; then
pid="$(pgrep -f 'SharpMailserver\.Server' | head -n 1 || true)"
fi
if [ -n "${pid}" ] && [ -r "/proc/${pid}/status" ]; then
awk '/VmRSS:/ { printf "%.1f", $2 / 1024.0 }' "/proc/${pid}/status"
elif [ -n "${pid}" ] && command -v ps >/dev/null 2>&1; then
ps -o rss= -p "${pid}" 2>/dev/null | awk '{ printf "%.1f", $1 / 1024.0 }'
else
printf "n/a"
fi
}
CPU_MODEL="$(grep -m1 'model name' /proc/cpuinfo 2>/dev/null | cut -d: -f2- | xargs || sysctl -n machdep.cpu.brand_string 2>/dev/null || uname -m)"
VCPUS="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo '?')"
KERNEL="$(uname -sr)"
IDLE_RSS="$(read_rss_mb "${SERVER_PID}")"
echo "======================================================================"
echo " SharpMailserver Open Benchmark Suite v1.1.0"
echo "======================================================================"
echo " Timestamp (UTC): $(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo " CPU / vCPUs: ${CPU_MODEL} (${VCPUS} threads)"
echo " OS / Kernel: ${KERNEL}"
echo " Target SMTP: ${SMTP_HOST}:${SMTP_PORT} (${CONCURRENCY} workers, ${TOTAL_MESSAGES} msgs)"
echo " Target HTTP: ${HTTP_URL}"
echo " Idle Server RSS: ${IDLE_RSS} MB"
echo "======================================================================"
python3 - "${SMTP_HOST}" "${SMTP_PORT}" "${HTTP_URL}" "${RECIPIENT}" "${CONCURRENCY}" "${TOTAL_MESSAGES}" "${SEARCH_ITERATIONS}" "${ADMIN_USER}" "${ADMIN_PASS}" <<'PY'
import asyncio
import base64
import json
import statistics
import sys
import time
import urllib.request
smtp_host = sys.argv[1]
smtp_port = int(sys.argv[2])
http_url = sys.argv[3].rstrip("/")
recipient = sys.argv[4]
concurrency = int(sys.argv[5])
total_messages = int(sys.argv[6])
search_runs = int(sys.argv[7])
auth_user = sys.argv[8]
auth_pass = sys.argv[9]
zugferd_xml = (
''
''
'RE-2026-90421'
''
) * 35
zugferd_b64 = base64.encodebytes(zugferd_xml.encode("utf-8")).decode("ascii")
body_filler = ("SharpMailserver deterministic benchmark line for MIME + FTS indexing.\r\n" * 180)
def build_msg(seq: int) -> bytes:
with_invoice = (seq % 4 == 0)
boundary = f"smbench-{seq}"
headers = (
f"From: Bench Sender \r\n"
f"To: <{recipient}>\r\n"
f"Subject: Benchmark Run #{seq} Invoice RE-2026-{seq:05d}\r\n"
f"Message-ID: \r\n"
f"Date: Fri, 30 Oct 2026 10:00:00 +0000\r\n"
f"MIME-Version: 1.0\r\n"
)
if not with_invoice:
payload = headers + "Content-Type: text/plain; charset=utf-8\r\n\r\n" + body_filler + "\r\n.\r\n"
else:
payload = (
headers
+ f'Content-Type: multipart/mixed; boundary="{boundary}"\r\n\r\n'
+ f"--{boundary}\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n{body_filler}\r\n"
+ f"--{boundary}\r\nContent-Type: text/xml; name=\"factur-x.xml\"\r\n"
+ "Content-Transfer-Encoding: base64\r\n\r\n"
+ zugferd_b64
+ f"\r\n--{boundary}--\r\n.\r\n"
)
return payload.encode("utf-8")
async def smtp_worker(worker_id: int, count: int, latencies: list):
reader, writer = await asyncio.open_connection(smtp_host, smtp_port)
await reader.readline()
writer.write(b"EHLO bench.local\r\n")
await writer.drain()
while True:
line = await reader.readline()
if not line or line[3:4] == b" ":
break
base = worker_id * count
for i in range(count):
raw = build_msg(base + i)
t0 = time.perf_counter()
writer.write(b"MAIL FROM:\r\n")
await writer.drain()
r1 = await reader.readline()
writer.write(f"RCPT TO:<{recipient}>\r\n".encode("ascii"))
await writer.drain()
r2 = await reader.readline()
writer.write(b"DATA\r\n")
await writer.drain()
r3 = await reader.readline()
if not r3.startswith(b"354"):
raise RuntimeError(f"Unexpected DATA reply: {r3!r} (MAIL={r1!r}, RCPT={r2!r})")
writer.write(raw)
await writer.drain()
r4 = await reader.readline()
if not r4.startswith(b"250"):
raise RuntimeError(f"Delivery failed: {r4!r}")
latencies.append((time.perf_counter() - t0) * 1000.0)
writer.write(b"QUIT\r\n")
await writer.drain()
writer.close()
def pct(sorted_vals, p):
if not sorted_vals:
return 0.0
idx = min(len(sorted_vals) - 1, max(0, int(round((p / 100.0) * (len(sorted_vals) - 1)))))
return sorted_vals[idx]
async def main():
per_worker = max(1, total_messages // concurrency)
actual_total = per_worker * concurrency
latencies = []
print(f"\n[1/2] Running ESMTP Inbound Load ({actual_total} messages across {concurrency} TCP streams)...")
t_start = time.perf_counter()
await asyncio.gather(*(smtp_worker(w, per_worker, latencies) for w in range(concurrency)))
elapsed = time.perf_counter() - t_start
latencies.sort()
rate = actual_total / elapsed
print(f" -> Completed in: {elapsed:.2f} s")
print(f" -> Throughput: {rate:,.1f} messages/sec")
print(f" -> Latency p50: {pct(latencies, 50):.2f} ms")
print(f" -> Latency p95: {pct(latencies, 95):.2f} ms")
print(f" -> Latency p99: {pct(latencies, 99):.2f} ms")
if auth_user and auth_pass:
print(f"\n[2/2] Running Full-Text & ZUGFeRD Search Latency ({search_runs} queries)...")
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor())
login_req = urllib.request.Request(
f"{http_url}/api/auth/login",
data=json.dumps({"email": auth_user, "password": auth_pass}).encode("utf-8"),
headers={"Content-Type": "application/json"},
method="POST",
)
opener.open(login_req, timeout=10).read()
s_lat = []
for i in range(search_runs):
q = f"RE-2026-{(i * 4) % max(4, actual_total):05d}"
t0 = time.perf_counter()
with opener.open(f"{http_url}/api/mail/messages?folder=INBOX&q={q}&limit=25", timeout=10) as resp:
resp.read()
s_lat.append((time.perf_counter() - t0) * 1000.0)
s_lat.sort()
print(f" -> Search p50: {pct(s_lat, 50):.2f} ms")
print(f" -> Search p95: {pct(s_lat, 95):.2f} ms")
print(f" -> Search p99: {pct(s_lat, 99):.2f} ms")
else:
print("\n[2/2] Skipping authenticated search test (pass --user --password to measure search p95).")
asyncio.run(main())
PY
POST_RSS="$(read_rss_mb "${SERVER_PID}")"
echo "----------------------------------------------------------------------"
echo " Post-Run Server RSS: ${POST_RSS} MB (Idle baseline was: ${IDLE_RSS} MB)"
echo "======================================================================"