Dart TCP/heartbeat, Python sustained sampling, TypeScript WS 1MB 경로의 성능 병목을 분리하고 검증 산출물을 함께 남긴다.
765 lines
29 KiB
Python
765 lines
29 KiB
Python
"""Same-language Python stress / benchmark harness.
|
|
|
|
고성능 병렬 운용 기준선 Milestone의 lang-baseline Task에서, TypeScript stress.ts와 같은 출력
|
|
계약(ROW|/SKIP|/SUMMARY|)으로 Python same-language TCP/WS baseline을 남긴다. 절대 성능 합격선은
|
|
고정하지 않고 local 환경 baseline(throughput, p50/p95/p99 latency, peak RSS)을 기록하며, 안정성
|
|
합격선만 hard fail로 둔다: timeout 0, nonce mismatch 0, response type mismatch 0, connection별
|
|
FIFO 위반 0, 종료 후 pending leak 0.
|
|
|
|
실행: python3 bench/stress.py [--mode quick|full] [--transport tcp,ws] [--profiles a,b]
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import socket
|
|
import sys
|
|
import time
|
|
from contextlib import suppress
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from proto_socket.communicator import type_name_of
|
|
from proto_socket.packets.message_common_pb2 import TestData
|
|
from proto_socket.tcp_client import connect_tcp
|
|
from proto_socket.tcp_server import TcpServer
|
|
from proto_socket.ws_client import connect_ws
|
|
from proto_socket.ws_server import WsServer
|
|
|
|
HOST = "127.0.0.1"
|
|
LANGUAGE = "Python"
|
|
WS_PATH = "/"
|
|
ALL_PROFILES = ["roundtrip", "burst", "sustained", "parallel", "slow-mix", "payload"]
|
|
|
|
# payload size matrix 축. quick은 작은 샘플(1KB/64KB), full은 Milestone 기준(1KB/64KB/1MB)을 측정한다.
|
|
PAYLOAD_SEED = "0123456789abcdef"
|
|
PAYLOAD_SIZES_QUICK = [1024, 65536]
|
|
PAYLOAD_SIZES_FULL = [1024, 65536, 1048576]
|
|
|
|
_rows_emitted = 0
|
|
_total_violations = 0
|
|
|
|
|
|
class Stability:
|
|
"""안정성 위반 카운터. 0이 아니면 해당 row는 FAIL이고 프로세스도 non-zero exit한다."""
|
|
|
|
__slots__ = ("timeouts", "nonce_mismatch", "type_mismatch", "fifo_violations", "pending_leak")
|
|
|
|
def __init__(self) -> None:
|
|
self.timeouts = 0
|
|
self.nonce_mismatch = 0
|
|
self.type_mismatch = 0
|
|
self.fifo_violations = 0
|
|
self.pending_leak = 0
|
|
|
|
def violations(self) -> int:
|
|
return (
|
|
self.timeouts
|
|
+ self.nonce_mismatch
|
|
+ self.type_mismatch
|
|
+ self.fifo_violations
|
|
+ self.pending_leak
|
|
)
|
|
|
|
|
|
def parser_map():
|
|
return {type_name_of(TestData): TestData.FromString}
|
|
|
|
|
|
def log(line: str) -> None:
|
|
print(line, file=sys.stderr, flush=True)
|
|
|
|
|
|
def mem_mb() -> float:
|
|
try:
|
|
with open("/proc/self/statm", encoding="ascii") as handle:
|
|
resident_pages = int(handle.read().split()[1])
|
|
return resident_pages * 4096 / (1024 * 1024)
|
|
except OSError:
|
|
import resource
|
|
|
|
return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024
|
|
|
|
|
|
def free_port() -> int:
|
|
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
try:
|
|
sock.bind((HOST, 0))
|
|
return sock.getsockname()[1]
|
|
finally:
|
|
sock.close()
|
|
|
|
|
|
def test_data_payload_bytes(message: str) -> int:
|
|
return len(TestData(index=0, message=message).SerializeToString())
|
|
|
|
|
|
def payload_filler(size: int) -> str:
|
|
"""size 바이트 길이의 deterministic filler string을 만든다."""
|
|
if size <= 0:
|
|
return ""
|
|
reps = -(-size // len(PAYLOAD_SEED))
|
|
return (PAYLOAD_SEED * reps)[:size]
|
|
|
|
|
|
def payload_label(num_bytes: int) -> str:
|
|
"""payload 크기를 사람이 읽는 축 이름으로 바꾼다(1024 -> 1KB, 1048576 -> 1MB)."""
|
|
if num_bytes >= 1048576 and num_bytes % 1048576 == 0:
|
|
return f"{num_bytes // 1048576}MB"
|
|
if num_bytes >= 1024 and num_bytes % 1024 == 0:
|
|
return f"{num_bytes // 1024}KB"
|
|
return f"{num_bytes}B"
|
|
|
|
|
|
def percentile(sorted_values: list[float], p: float) -> float:
|
|
if not sorted_values:
|
|
return 0.0
|
|
rank = max(0, min(len(sorted_values) - 1, -(-int(p * len(sorted_values)) // 100) - 1))
|
|
return sorted_values[rank]
|
|
|
|
|
|
# Bounded reservoir sampler for sustained profile to prevent unbounded memory growth.
|
|
# During long sustained runs (e.g. 30 minutes) the number of requests can reach tens of millions.
|
|
# Storing all latencies would inflate the memory peak for reasons unrelated to implementation leaks.
|
|
# The reservoir sampler keeps at most SUSTAINED_MAX_LATENCY_SAMPLES samples while preserving
|
|
# statistical representativeness, and the request_count parameter tells summarize() how many
|
|
# total requests were actually processed so that throughput is computed from the real count.
|
|
SUSTAINED_MAX_LATENCY_SAMPLES = 100_000
|
|
|
|
|
|
class LatencyReservoir:
|
|
"""Deterministic reservoir sampler that keeps at most max_samples values."""
|
|
|
|
__slots__ = ("max_samples", "count", "samples")
|
|
|
|
def __init__(self, max_samples: int) -> None:
|
|
self.max_samples = max_samples
|
|
self.count = 0
|
|
self.samples: list[float] = []
|
|
|
|
def add(self, value: float) -> None:
|
|
self.count += 1
|
|
if len(self.samples) < self.max_samples:
|
|
self.samples.append(value)
|
|
return
|
|
slot = (self.count * 1103515245 + 12345) % self.count
|
|
if slot < self.max_samples:
|
|
self.samples[slot] = value
|
|
|
|
|
|
def classify_request_error(message: str, stability: Stability) -> None:
|
|
if "timeout" in message:
|
|
stability.timeouts += 1
|
|
elif "type mismatch" in message:
|
|
stability.type_mismatch += 1
|
|
else:
|
|
stability.nonce_mismatch += 1
|
|
|
|
|
|
def emit_row(
|
|
*,
|
|
profile: str,
|
|
axis: str,
|
|
transport: str,
|
|
payload_bytes: int,
|
|
client_count: int,
|
|
requests: int,
|
|
throughput: float,
|
|
p50: float,
|
|
p95: float,
|
|
p99: float,
|
|
mem: float,
|
|
stability: Stability,
|
|
) -> None:
|
|
global _rows_emitted, _total_violations
|
|
violations = stability.violations()
|
|
status = "PASS" if violations == 0 else "FAIL"
|
|
_total_violations += violations
|
|
_rows_emitted += 1
|
|
fields = [
|
|
"ROW",
|
|
profile,
|
|
axis,
|
|
LANGUAGE,
|
|
transport,
|
|
str(payload_bytes),
|
|
str(client_count),
|
|
str(requests),
|
|
f"{throughput:.1f}",
|
|
f"{p50:.3f}",
|
|
f"{p95:.3f}",
|
|
f"{p99:.3f}",
|
|
str(stability.timeouts),
|
|
str(stability.nonce_mismatch),
|
|
str(stability.type_mismatch),
|
|
str(stability.fifo_violations),
|
|
str(stability.pending_leak),
|
|
"0", # queueBacklog: same-language baseline에는 inbound gateway가 없다.
|
|
"0", # gatewayBacklog
|
|
"off",
|
|
f"{mem:.1f}",
|
|
status,
|
|
]
|
|
print("|".join(fields), flush=True)
|
|
|
|
|
|
def emit_skip(profile: str, axis: str, transport: str, reason: str) -> None:
|
|
print(f"SKIP|{profile}|{axis}|{LANGUAGE}|{transport}|{reason}", flush=True)
|
|
|
|
|
|
def summarize(
|
|
profile: str,
|
|
axis: str,
|
|
transport: str,
|
|
latencies: list[float],
|
|
elapsed_ms: float,
|
|
client_count: int,
|
|
stability: Stability,
|
|
mem: float,
|
|
*,
|
|
request_count: int | None = None,
|
|
) -> None:
|
|
requests = request_count if request_count is not None else len(latencies)
|
|
throughput = (requests / elapsed_ms * 1000.0) if elapsed_ms > 0 else 0.0
|
|
emit_row(
|
|
profile=profile,
|
|
axis=axis,
|
|
transport=transport,
|
|
payload_bytes=test_data_payload_bytes("req-0"),
|
|
client_count=client_count,
|
|
requests=requests,
|
|
throughput=throughput,
|
|
p50=percentile(list(sorted(latencies)), 50),
|
|
p95=percentile(list(sorted(latencies)), 95),
|
|
p99=percentile(list(sorted(latencies)), 99),
|
|
mem=mem,
|
|
stability=stability,
|
|
)
|
|
|
|
|
|
def emit_throughput(
|
|
profile: str,
|
|
axis: str,
|
|
transport: str,
|
|
count: int,
|
|
elapsed_ms: float,
|
|
client_count: int,
|
|
payload_bytes: int,
|
|
stability: Stability,
|
|
mem: float,
|
|
) -> None:
|
|
throughput = (count / elapsed_ms * 1000.0) if elapsed_ms > 0 else 0.0
|
|
emit_row(
|
|
profile=profile,
|
|
axis=axis,
|
|
transport=transport,
|
|
payload_bytes=payload_bytes,
|
|
client_count=client_count,
|
|
requests=count,
|
|
throughput=throughput,
|
|
p50=0.0,
|
|
p95=0.0,
|
|
p99=0.0,
|
|
mem=mem,
|
|
stability=stability,
|
|
)
|
|
|
|
|
|
def make_server(transport: str, port: int, on_connected):
|
|
if transport == "tcp":
|
|
server = TcpServer(HOST, port, interval_sec=0, wait_sec=0, parser_map=parser_map())
|
|
elif transport == "ws":
|
|
server = WsServer(HOST, port, WS_PATH, interval_sec=0, wait_sec=0, parser_map=parser_map())
|
|
else:
|
|
raise ValueError(f"unknown transport {transport!r}")
|
|
server.on_client_connected = on_connected
|
|
return server
|
|
|
|
|
|
async def dial(transport: str, port: int):
|
|
if transport == "tcp":
|
|
return await connect_tcp(HOST, port, 0, 0, parser_map())
|
|
if transport == "ws":
|
|
return await connect_ws(HOST, port, WS_PATH, 0, 0, parser_map())
|
|
raise ValueError(f"unknown transport {transport!r}")
|
|
|
|
|
|
async def dial_with_retry(transport: str, port: int):
|
|
loop = asyncio.get_running_loop()
|
|
deadline = loop.time() + 3.0
|
|
last_error: Exception | None = None
|
|
while loop.time() < deadline:
|
|
try:
|
|
return await asyncio.wait_for(dial(transport, port), 0.3)
|
|
except Exception as exc: # noqa: BLE001
|
|
last_error = exc
|
|
await asyncio.sleep(0.05)
|
|
raise TimeoutError(f"connect {transport}:{port} timed out: {last_error}")
|
|
|
|
|
|
def make_echo_server(transport: str, port: int):
|
|
# request-response 축에서는 client가 concurrency C로 동시에 send하므로 frame 도착 순서가
|
|
# 결정적이지 않다. 따라서 여기서는 FIFO를 측정하지 않고 응답 정확성/timeout만 본다. per-connection
|
|
# FIFO는 단일 connection 순차 send인 burst 축에서 검증한다.
|
|
return make_server(
|
|
transport,
|
|
port,
|
|
lambda client: client.communicator.add_request_listener(
|
|
type_name_of(TestData),
|
|
lambda req: TestData(index=req.index * 2, message="echo:" + req.message),
|
|
),
|
|
)
|
|
|
|
|
|
def make_slow_mix_server(transport: str, port: int, slow_every: int, delay: float):
|
|
async def on_request(req: TestData) -> TestData:
|
|
if (req.index + 1) % slow_every == 0:
|
|
await asyncio.sleep(delay)
|
|
return TestData(index=req.index * 2, message="echo:" + req.message)
|
|
|
|
return make_server(
|
|
transport,
|
|
port,
|
|
lambda client: client.communicator.add_request_listener(type_name_of(TestData), on_request),
|
|
)
|
|
|
|
|
|
async def run_request_load(client, total: int, concurrency: int, timeout: float, stability: Stability) -> list[float]:
|
|
latencies: list[float] = []
|
|
issued = 0
|
|
next_index = 0
|
|
|
|
async def one_request(index: int) -> None:
|
|
started = time.perf_counter()
|
|
try:
|
|
res = await client.communicator.send_request(
|
|
TestData(index=index, message=f"req-{index}"), TestData, timeout=timeout
|
|
)
|
|
latencies.append((time.perf_counter() - started) * 1000.0)
|
|
if res.index != index * 2 or res.message != f"echo:req-{index}":
|
|
stability.nonce_mismatch += 1
|
|
except Exception as exc: # noqa: BLE001
|
|
classify_request_error(str(exc), stability)
|
|
|
|
while issued < total:
|
|
batch_size = min(concurrency, total - issued)
|
|
indices = list(range(next_index, next_index + batch_size))
|
|
next_index += batch_size
|
|
issued += batch_size
|
|
await asyncio.gather(*(one_request(i) for i in indices))
|
|
|
|
return latencies
|
|
|
|
|
|
async def profile_roundtrip(transport: str, mode: str) -> None:
|
|
concurrencies = [1, 16, 64, 256]
|
|
batches = 2 if mode == "quick" else 20
|
|
timeout = 5.0 if mode == "quick" else 15.0
|
|
log(f"[roundtrip] transport={transport} mode={mode} concurrencies=1,16,64,256 batches/level={batches}")
|
|
for concurrency in concurrencies:
|
|
stability = Stability()
|
|
total = concurrency * batches
|
|
port = free_port()
|
|
server = make_echo_server(transport, port)
|
|
await server.start()
|
|
try:
|
|
client = await dial_with_retry(transport, port)
|
|
try:
|
|
started = time.perf_counter()
|
|
latencies = await run_request_load(client, total, concurrency, timeout, stability)
|
|
elapsed = (time.perf_counter() - started) * 1000.0
|
|
summarize("roundtrip", f"concurrency={concurrency}", transport, latencies, elapsed, 1, stability, mem_mb())
|
|
finally:
|
|
with suppress(Exception):
|
|
await asyncio.wait_for(client.close(), 2.0)
|
|
if client.communicator.is_alive():
|
|
stability.pending_leak += 1
|
|
except Exception as exc: # noqa: BLE001
|
|
stability.timeouts += 1
|
|
log(f"[roundtrip] concurrency={concurrency} error={exc}")
|
|
summarize("roundtrip", f"concurrency={concurrency}", transport, [], 0.0, 1, stability, mem_mb())
|
|
finally:
|
|
await server.stop()
|
|
log(f"[roundtrip] transport={transport} concurrency={concurrency} done violations={stability.violations()}")
|
|
|
|
|
|
async def profile_burst(transport: str, mode: str) -> None:
|
|
counts = [200] if mode == "quick" else [1000, 10000, 100000]
|
|
log(f"[burst] transport={transport} mode={mode} counts={counts}")
|
|
for count in counts:
|
|
stability = Stability()
|
|
state = {"received": 0, "last_index": -1}
|
|
loop = asyncio.get_running_loop()
|
|
done = loop.create_future()
|
|
|
|
def on_connected(client):
|
|
def on_message(data):
|
|
if data.index <= state["last_index"]:
|
|
stability.fifo_violations += 1
|
|
state["last_index"] = data.index
|
|
state["received"] += 1
|
|
if state["received"] >= count and not done.done():
|
|
done.set_result(None)
|
|
|
|
client.communicator.add_listener(type_name_of(TestData), on_message)
|
|
|
|
port = free_port()
|
|
server = make_server(transport, port, on_connected)
|
|
await server.start()
|
|
client = None
|
|
try:
|
|
client = await dial_with_retry(transport, port)
|
|
started = time.perf_counter()
|
|
for i in range(count):
|
|
await client.communicator.send(TestData(index=i, message=f"b-{i}"))
|
|
try:
|
|
await asyncio.wait_for(done, 10.0 if mode == "quick" else 60.0)
|
|
except TimeoutError:
|
|
stability.timeouts += 1
|
|
log(f"[burst] count={count} dispatch timeout received={state['received']}")
|
|
elapsed = (time.perf_counter() - started) * 1000.0
|
|
if state["received"] != count:
|
|
stability.pending_leak += 1
|
|
emit_throughput("burst", f"count={count}", transport, count, elapsed, 1, test_data_payload_bytes("b-0"), stability, mem_mb())
|
|
except Exception as exc: # noqa: BLE001
|
|
stability.timeouts += 1
|
|
log(f"[burst] count={count} error={exc}")
|
|
emit_throughput("burst", f"count={count}", transport, state["received"], 0.0, 1, test_data_payload_bytes("b-0"), stability, mem_mb())
|
|
finally:
|
|
if client is not None:
|
|
with suppress(Exception):
|
|
await asyncio.wait_for(client.close(), 2.0)
|
|
if client.communicator.is_alive():
|
|
stability.pending_leak += 1
|
|
await server.stop()
|
|
log(f"[burst] transport={transport} count={count} received={state['received']} violations={stability.violations()}")
|
|
|
|
|
|
async def profile_sustained(transport: str, mode: str) -> None:
|
|
durations_ms = [2000] if mode == "quick" else [30000, 300000, 1800000]
|
|
concurrency = 16
|
|
timeout = 15.0
|
|
log(f"[sustained] transport={transport} mode={mode} durations={durations_ms}ms concurrency={concurrency}")
|
|
for duration_ms in durations_ms:
|
|
stability = Stability()
|
|
peak = mem_mb()
|
|
port = free_port()
|
|
server = make_echo_server(transport, port)
|
|
await server.start()
|
|
try:
|
|
client = await dial_with_retry(transport, port)
|
|
sampler = LatencyReservoir(SUSTAINED_MAX_LATENCY_SAMPLES)
|
|
next_index = 0
|
|
loop = asyncio.get_running_loop()
|
|
deadline = loop.time() + duration_ms / 1000.0
|
|
try:
|
|
while loop.time() < deadline:
|
|
indices = list(range(next_index, next_index + concurrency))
|
|
next_index += concurrency
|
|
|
|
async def one(index: int) -> None:
|
|
started = time.perf_counter()
|
|
try:
|
|
res = await client.communicator.send_request(
|
|
TestData(index=index, message=f"s-{index}"), TestData, timeout=timeout
|
|
)
|
|
sampler.add((time.perf_counter() - started) * 1000.0)
|
|
if res.index != index * 2 or res.message != f"echo:s-{index}":
|
|
stability.nonce_mismatch += 1
|
|
except Exception as exc: # noqa: BLE001
|
|
classify_request_error(str(exc), stability)
|
|
|
|
await asyncio.gather(*(one(i) for i in indices))
|
|
current = mem_mb()
|
|
if current > peak:
|
|
peak = current
|
|
summarize("sustained", f"duration={duration_ms}ms", transport, sampler.samples, float(duration_ms), 1, stability, peak, request_count=sampler.count)
|
|
finally:
|
|
with suppress(Exception):
|
|
await asyncio.wait_for(client.close(), 2.0)
|
|
if client.communicator.is_alive():
|
|
stability.pending_leak += 1
|
|
except Exception as exc: # noqa: BLE001
|
|
stability.timeouts += 1
|
|
log(f"[sustained] duration={duration_ms}ms error={exc}")
|
|
summarize("sustained", f"duration={duration_ms}ms", transport, [], 0.0, 1, stability, peak, request_count=0)
|
|
finally:
|
|
await server.stop()
|
|
log(f"[sustained] transport={transport} duration={duration_ms}ms done peakMemMb={peak:.1f} violations={stability.violations()}")
|
|
|
|
|
|
async def profile_parallel(transport: str, mode: str) -> None:
|
|
client_counts = [4] if mode == "quick" else [16, 128, 512, 1024]
|
|
per_client = 50 if mode == "quick" else 500
|
|
concurrency = 8
|
|
timeout = 15.0
|
|
log(f"[parallel] transport={transport} mode={mode} clients={client_counts} perClient={per_client}")
|
|
for client_count in client_counts:
|
|
stability = Stability()
|
|
port = free_port()
|
|
server = make_echo_server(transport, port)
|
|
await server.start()
|
|
clients: list = []
|
|
try:
|
|
for _ in range(client_count):
|
|
try:
|
|
clients.append(await dial_with_retry(transport, port))
|
|
except Exception as exc: # noqa: BLE001
|
|
stability.timeouts += 1
|
|
log(f"[parallel] clients={client_count} dial error={exc}")
|
|
all_latencies: list[float] = []
|
|
started = time.perf_counter()
|
|
|
|
async def run_one(client) -> None:
|
|
all_latencies.extend(await run_request_load(client, per_client, concurrency, timeout, stability))
|
|
|
|
await asyncio.gather(*(run_one(c) for c in clients))
|
|
elapsed = (time.perf_counter() - started) * 1000.0
|
|
summarize("parallel", f"clients={client_count}", transport, all_latencies, elapsed, client_count, stability, mem_mb())
|
|
finally:
|
|
for client in clients:
|
|
with suppress(Exception):
|
|
await asyncio.wait_for(client.close(), 2.0)
|
|
if client.communicator.is_alive():
|
|
stability.pending_leak += 1
|
|
await server.stop()
|
|
log(f"[parallel] transport={transport} clients={client_count} done violations={stability.violations()}")
|
|
|
|
|
|
async def profile_slow_mix(transport: str, mode: str) -> None:
|
|
client_count = 4 if mode == "quick" else 16
|
|
per_client = 20 if mode == "quick" else 200
|
|
slow_every = 5 if mode == "quick" else 10
|
|
delay_ms = 50 if mode == "quick" else 100
|
|
timeout = 10.0 if mode == "quick" else 30.0
|
|
axis = f"clients={client_count},slowEvery={slow_every},delayMs={delay_ms}"
|
|
stability = Stability()
|
|
peak = mem_mb()
|
|
latencies: list[float] = []
|
|
elapsed = 0.0
|
|
log(f"[slow-mix] transport={transport} mode={mode} {axis} perClient={per_client}")
|
|
|
|
port = free_port()
|
|
server = make_slow_mix_server(transport, port, slow_every, delay_ms / 1000.0)
|
|
await server.start()
|
|
clients: list[tuple[int, object]] = []
|
|
try:
|
|
for client_id in range(client_count):
|
|
try:
|
|
clients.append((client_id, await dial_with_retry(transport, port)))
|
|
except Exception as exc: # noqa: BLE001
|
|
stability.timeouts += 1
|
|
log(f"[slow-mix] client={client_id} dial error={exc}")
|
|
|
|
completed_by_client = [-1] * client_count
|
|
started = time.perf_counter()
|
|
|
|
async def one_request(client, client_id: int, index: int) -> None:
|
|
message = f"sm-{client_id}-{index}"
|
|
req_start = time.perf_counter()
|
|
try:
|
|
res = await client.communicator.send_request(
|
|
TestData(index=index, message=message), TestData, timeout=timeout
|
|
)
|
|
latencies.append((time.perf_counter() - req_start) * 1000.0)
|
|
if res.index != index * 2 or res.message != "echo:" + message:
|
|
stability.nonce_mismatch += 1
|
|
if index <= completed_by_client[client_id]:
|
|
stability.fifo_violations += 1
|
|
completed_by_client[client_id] = index
|
|
except Exception as exc: # noqa: BLE001
|
|
classify_request_error(str(exc), stability)
|
|
|
|
await asyncio.gather(
|
|
*(
|
|
one_request(client, client_id, index)
|
|
for client_id, client in clients
|
|
for index in range(per_client)
|
|
)
|
|
)
|
|
peak = max(peak, mem_mb())
|
|
elapsed = (time.perf_counter() - started) * 1000.0
|
|
finally:
|
|
for _, client in clients:
|
|
with suppress(Exception):
|
|
await asyncio.wait_for(client.close(), 2.0)
|
|
if client.communicator.is_alive():
|
|
stability.pending_leak += 1
|
|
await server.stop()
|
|
ordered = sorted(latencies)
|
|
throughput = (len(latencies) / elapsed * 1000.0) if elapsed > 0 else 0.0
|
|
emit_row(
|
|
profile="slow-mix",
|
|
axis=axis,
|
|
transport=transport,
|
|
payload_bytes=test_data_payload_bytes("sm-0-0"),
|
|
client_count=client_count,
|
|
requests=len(latencies),
|
|
throughput=throughput,
|
|
p50=percentile(ordered, 50),
|
|
p95=percentile(ordered, 95),
|
|
p99=percentile(ordered, 99),
|
|
mem=peak,
|
|
stability=stability,
|
|
)
|
|
log(
|
|
f"[slow-mix] transport={transport} clients={client_count} done "
|
|
f"peakMemMb={peak:.1f} violations={stability.violations()}"
|
|
)
|
|
|
|
|
|
async def profile_payload(transport: str, mode: str) -> None:
|
|
sizes = PAYLOAD_SIZES_QUICK if mode == "quick" else PAYLOAD_SIZES_FULL
|
|
concurrency = 4 if mode == "quick" else 8
|
|
requests = 20 if mode == "quick" else 100
|
|
timeout = 10.0 if mode == "quick" else 30.0
|
|
log(
|
|
f"[payload] transport={transport} mode={mode} sizes={[payload_label(s) for s in sizes]} "
|
|
f"concurrency={concurrency} requests/size={requests}"
|
|
)
|
|
for size in sizes:
|
|
stability = Stability()
|
|
filler = payload_filler(size)
|
|
expected_echo = "echo:" + filler
|
|
payload_bytes = test_data_payload_bytes(filler)
|
|
axis = f"payload={payload_label(size)}"
|
|
peak = mem_mb()
|
|
port = free_port()
|
|
server = make_echo_server(transport, port)
|
|
await server.start()
|
|
try:
|
|
client = await dial_with_retry(transport, port)
|
|
try:
|
|
latencies: list[float] = []
|
|
issued = 0
|
|
next_index = 0
|
|
started = time.perf_counter()
|
|
|
|
async def one(index: int) -> None:
|
|
req_start = time.perf_counter()
|
|
try:
|
|
res = await client.communicator.send_request(
|
|
TestData(index=index, message=filler), TestData, timeout=timeout
|
|
)
|
|
latencies.append((time.perf_counter() - req_start) * 1000.0)
|
|
if res.index != index * 2 or res.message != expected_echo:
|
|
stability.nonce_mismatch += 1
|
|
except Exception as exc: # noqa: BLE001
|
|
classify_request_error(str(exc), stability)
|
|
|
|
while issued < requests:
|
|
batch_size = min(concurrency, requests - issued)
|
|
indices = list(range(next_index, next_index + batch_size))
|
|
next_index += batch_size
|
|
issued += batch_size
|
|
await asyncio.gather(*(one(i) for i in indices))
|
|
current = mem_mb()
|
|
if current > peak:
|
|
peak = current
|
|
elapsed = (time.perf_counter() - started) * 1000.0
|
|
ordered = sorted(latencies)
|
|
throughput = (len(latencies) / elapsed * 1000.0) if elapsed > 0 else 0.0
|
|
emit_row(
|
|
profile="payload",
|
|
axis=axis,
|
|
transport=transport,
|
|
payload_bytes=payload_bytes,
|
|
client_count=1,
|
|
requests=len(latencies),
|
|
throughput=throughput,
|
|
p50=percentile(ordered, 50),
|
|
p95=percentile(ordered, 95),
|
|
p99=percentile(ordered, 99),
|
|
mem=peak,
|
|
stability=stability,
|
|
)
|
|
finally:
|
|
with suppress(Exception):
|
|
await asyncio.wait_for(client.close(), 2.0)
|
|
if client.communicator.is_alive():
|
|
stability.pending_leak += 1
|
|
except Exception as exc: # noqa: BLE001
|
|
stability.timeouts += 1
|
|
log(f"[payload] size={payload_label(size)} error={exc}")
|
|
emit_row(
|
|
profile="payload",
|
|
axis=axis,
|
|
transport=transport,
|
|
payload_bytes=payload_bytes,
|
|
client_count=1,
|
|
requests=0,
|
|
throughput=0.0,
|
|
p50=0.0,
|
|
p95=0.0,
|
|
p99=0.0,
|
|
mem=peak,
|
|
stability=stability,
|
|
)
|
|
finally:
|
|
await server.stop()
|
|
log(
|
|
f"[payload] transport={transport} size={payload_label(size)} bytes={payload_bytes} "
|
|
f"peakMemMb={peak:.1f} violations={stability.violations()}"
|
|
)
|
|
|
|
|
|
RUNNERS = {
|
|
"roundtrip": profile_roundtrip,
|
|
"burst": profile_burst,
|
|
"sustained": profile_sustained,
|
|
"parallel": profile_parallel,
|
|
"slow-mix": profile_slow_mix,
|
|
"payload": profile_payload,
|
|
}
|
|
|
|
|
|
def parse_list(value: str | None, default: list[str]) -> list[str]:
|
|
if not value:
|
|
return default
|
|
items = [v.strip() for v in value.split(",") if v.strip()]
|
|
return items or default
|
|
|
|
|
|
async def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--mode", choices=["quick", "full"], default="quick")
|
|
parser.add_argument("--quick", action="store_const", const="quick", dest="mode")
|
|
parser.add_argument("--full", action="store_const", const="full", dest="mode")
|
|
parser.add_argument("--transport", "--transports", dest="transport", default=None)
|
|
parser.add_argument("--profiles", "--profile", dest="profiles", default=None)
|
|
args = parser.parse_args()
|
|
|
|
mode = args.mode or "quick"
|
|
transports = parse_list(args.transport, ["tcp", "ws"])
|
|
profiles = parse_list(args.profiles, ALL_PROFILES)
|
|
|
|
log(
|
|
f"INFO stress harness language={LANGUAGE} mode={mode} transports={','.join(transports)} "
|
|
f"profiles={','.join(profiles)} typeName={type_name_of(TestData)}"
|
|
)
|
|
|
|
for transport in transports:
|
|
if transport not in ("tcp", "ws"):
|
|
emit_skip("all", "transport", transport, "unsupported transport for Python same-language baseline")
|
|
continue
|
|
for profile in profiles:
|
|
runner = RUNNERS.get(profile)
|
|
if runner is None:
|
|
emit_skip(profile, "profile", transport, "profile not part of Python same-language baseline (gateway is TypeScript-specific)")
|
|
continue
|
|
await runner(transport, mode)
|
|
|
|
status = "PASS" if _total_violations == 0 else "FAIL"
|
|
print(
|
|
f"SUMMARY|status={status}|language={LANGUAGE}|mode={mode}|transports={','.join(transports)}|"
|
|
f"profiles={','.join(profiles)}|rows={_rows_emitted}|stability_violations={_total_violations}",
|
|
flush=True,
|
|
)
|
|
if status != "PASS":
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|