hermes-agent/scripts/benchmark_click_paths.py

"""
Benchmark: CDP coordinate click (real Lightpanda) vs agent-browser IPC latency.

This measures:
1. CDP path — browser_click(x, y) hitting real Lightpanda WS at :63372
2. agent-browser HTTP IPC — raw HTTP request to /api/click (measures the IPC
   channel cost without needing a loaded page)

Usage: python scripts/benchmark_click_paths.py [--iterations N] [--warmup N]
"""
from __future__ import annotations

import argparse
import json
import sys
import time
import urllib.request
from statistics import mean, median, stdev
from typing import List, Dict, Tuple

sys.path.insert(0, "/private/tmp/hermes-coord-click")

LIGHTPANDA_WS = "ws://127.0.0.1:63372/"
AGENT_BROWSER_BASE = "http://127.0.0.1:63371"


def _stats(times_s: List[float]) -> Dict[str, float]:
    ms = [t * 1000 for t in times_s]
    return {
        "mean_ms": mean(ms),
        "median_ms": median(ms),
        "min_ms": min(ms),
        "max_ms": max(ms),
        "stdev_ms": stdev(ms) if len(ms) > 1 else 0.0,
        "p95_ms": sorted(ms)[int(len(ms) * 0.95)],
    }


def _row(label: str, stats: Dict, col_w: int = 9) -> None:
    print(
        f"  {label:<44}  "
        f"{stats['mean_ms']:>{col_w}.2f}  "
        f"{stats['median_ms']:>{col_w}.2f}  "
        f"{stats['min_ms']:>{col_w}.2f}  "
        f"{stats['p95_ms']:>{col_w}.2f}  "
        f"{stats['max_ms']:>{col_w}.2f}  ms"
    )


def _bench(fn, n: int) -> Tuple[List[float], int]:
    times, errors = [], 0
    for _ in range(n):
        t0 = time.perf_counter()
        try:
            result = fn()
            elapsed = time.perf_counter() - t0
            if isinstance(result, str):
                d = json.loads(result)
                if not d.get("success"):
                    errors += 1
        except Exception:
            elapsed = time.perf_counter() - t0
            errors += 1
        times.append(elapsed)
    return times, errors


def _http_get(url: str) -> float:
    """Return elapsed seconds for a single HTTP GET (measures IPC round-trip)."""
    t0 = time.perf_counter()
    try:
        with urllib.request.urlopen(url, timeout=5) as r:
            r.read()
    except Exception:
        pass
    return time.perf_counter() - t0


def run_benchmark(iterations: int = 200, warmup: int = 15) -> None:
    print(f"\n{'=' * 76}")
    print(f"  browser_click: CDP Coordinate Click (real Lightpanda) Benchmark")
    print(f"{'=' * 76}")
    print(f"  Iterations: {iterations}  |  Warmup: {warmup}")

    import importlib
    import tools.browser_tool as bt
    import tools.browser_cdp_tool as cdp_mod
    importlib.reload(cdp_mod)
    importlib.reload(bt)

    bt._is_camofox_mode = lambda: False
    _orig_resolve = cdp_mod._resolve_cdp_endpoint

    # -----------------------------------------------------------------------
    # A. CDP coord click via real Lightpanda WS
    # -----------------------------------------------------------------------
    print(f"\n  [A] CDP coord → Lightpanda WS ({LIGHTPANDA_WS})")
    cdp_mod._resolve_cdp_endpoint = lambda: LIGHTPANDA_WS

    print(f"      Warming up ({warmup})...")
    for _ in range(warmup):
        bt.browser_click(x=100.0, y=100.0, task_id="bench")

    print(f"      Benchmarking ({iterations})...")
    cdp_times, cdp_err = _bench(
        lambda: bt.browser_click(x=150.0, y=200.0, task_id="bench"),
        iterations,
    )
    cdp_mod._resolve_cdp_endpoint = _orig_resolve
    cdp_stats = _stats(cdp_times)
    print(f"      Done — {cdp_err} errors, mean={cdp_stats['mean_ms']:.2f}ms")

    # -----------------------------------------------------------------------
    # B. agent-browser HTTP IPC latency (GET /api/sessions — lightweight ping)
    # -----------------------------------------------------------------------
    print(f"\n  [B] agent-browser HTTP IPC round-trip (:63371/api/sessions)")
    print(f"      Warming up ({warmup})...")
    for _ in range(warmup):
        _http_get(f"{AGENT_BROWSER_BASE}/api/sessions")

    print(f"      Benchmarking ({iterations})...")
    ab_times = []
    for _ in range(iterations):
        ab_times.append(_http_get(f"{AGENT_BROWSER_BASE}/api/sessions"))
    ab_stats = _stats(ab_times)
    print(f"      Done — mean={ab_stats['mean_ms']:.2f}ms")

    # -----------------------------------------------------------------------
    # C. Raw Lightpanda WS latency (single CDP call — no click, just a ping)
    #    This measures WS connection setup + 1 message round-trip
    # -----------------------------------------------------------------------
    print(f"\n  [C] Raw single CDP call to Lightpanda (Target.getTargets baseline)")
    print(f"      Warming up ({warmup})...")

    async def _single_cdp():
        from tools.browser_cdp_tool import _cdp_call, _run_async
        return _run_async(_cdp_call(LIGHTPANDA_WS, "Target.getTargets", {}, None, 5.0))

    def _time_single_cdp():
        from tools.browser_cdp_tool import _cdp_call, _run_async
        return _run_async(_cdp_call(LIGHTPANDA_WS, "Target.getTargets", {}, None, 5.0))

    for _ in range(warmup):
        _time_single_cdp()

    print(f"      Benchmarking ({iterations})...")
    single_cdp_times = []
    for _ in range(iterations):
        t0 = time.perf_counter()
        _time_single_cdp()
        single_cdp_times.append(time.perf_counter() - t0)
    single_cdp_stats = _stats(single_cdp_times)
    print(f"      Done — mean={single_cdp_stats['mean_ms']:.2f}ms per WS connection+call")

    # -----------------------------------------------------------------------
    # Results
    # -----------------------------------------------------------------------
    col_w = 9
    print(f"\n{'─' * 76}")
    print(f"  {'Path':<44}  {'Mean':>{col_w}}  {'Median':>{col_w}}  {'Min':>{col_w}}  {'p95':>{col_w}}  {'Max':>{col_w}}")
    print(f"{'─' * 76}")
    _row("CDP coord (x,y) → Lightpanda [3 WS conns]", cdp_stats, col_w)
    _row("Single CDP call [1 WS conn baseline]", single_cdp_stats, col_w)
    _row("agent-browser HTTP IPC [1 request]", ab_stats, col_w)
    print(f"{'─' * 76}")

    expected_3x = single_cdp_stats["mean_ms"] * 3
    print(f"\n  Analysis:")
    print(f"    • Full CDP click = {cdp_stats['mean_ms']:.2f}ms  ({cdp_stats['mean_ms'] / single_cdp_stats['mean_ms']:.1f}× single call)")
    print(f"    • Expected at 3 sequential WS connections: ~{expected_3x:.1f}ms")
    print(f"    • Single WS conn+call baseline: {single_cdp_stats['mean_ms']:.2f}ms")
    print(f"    • agent-browser HTTP IPC: {ab_stats['mean_ms']:.2f}ms per call")
    print(f"      (ref click = ~1 IPC call, fallback mouse = ~3 IPC calls)")
    print(f"    • Estimated ref click latency: ~{ab_stats['mean_ms']:.1f}ms")
    print(f"    • Estimated fallback (mouse) latency: ~{ab_stats['mean_ms']*3:.1f}ms")
    print(f"    • CDP coord vs ref estimate: {cdp_stats['mean_ms']:.1f}ms vs ~{ab_stats['mean_ms']:.1f}ms")
    cdp_vs_ref_est = cdp_stats["mean_ms"] / ab_stats["mean_ms"]
    print(f"      Ratio: {cdp_vs_ref_est:.1f}x — overhead from 3 per-click WS conn setups.")
    print(f"    • Both are well under 100ms human perception threshold.")
    print(f"    • A persistent WS connection would reduce CDP clicks to ~{single_cdp_stats['mean_ms']*2:.1f}ms")
    print(f"      (just mousePressed + mouseReleased on existing session).")
    print()


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--iterations", type=int, default=200)
    parser.add_argument("--warmup", type=int, default=15)
    args = parser.parse_args()
    run_benchmark(iterations=args.iterations, warmup=args.warmup)