hermes-agent/scripts/benchmark_click_paths.py

"""
Benchmark: Current main (3 separate WS connections) vs optimized (1 connection).

Compares the two CDP coordinate click implementations against a real
Lightpanda WebSocket at ws://127.0.0.1:63372/.

  - Baseline (current main style): 3 separate _cdp_call() invocations, each
    opening a fresh WS connection (Target.getTargets, mousePressed, mouseReleased)
  - Optimized (this PR): single WS connection with all 4 messages pipelined
    (getTargets + attachToTarget + mousePressed+mouseReleased in one burst)

Also measures the agent-browser HTTP IPC round-trip as a reference point
for how fast the existing ref-based click path is.

Usage:
    python scripts/benchmark_click_paths.py
    python scripts/benchmark_click_paths.py --iterations 300 --warmup 20
"""
from __future__ import annotations

import argparse
import asyncio
import json
import sys
import time
import urllib.request
from statistics import mean, median, stdev
from typing import List, Dict, Optional, Tuple

sys.path.insert(0, "/private/tmp/hermes-coord-click")

LIGHTPANDA_WS = "ws://127.0.0.1:63372/"
AGENT_BROWSER_PORT = 63371


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

def _stats(times_s: List[float]) -> Dict:
    ms = [t * 1000 for t in times_s]
    return {
        "mean_ms":   mean(ms),
        "median_ms": median(ms),
        "min_ms":    min(ms),
        "max_ms":    max(ms),
        "stdev_ms":  stdev(ms) if len(ms) > 1 else 0.0,
        "p95_ms":    sorted(ms)[int(len(ms) * 0.95)],
    }


def _bench(fn, warmup: int, n: int) -> Tuple[List[float], int]:
    for _ in range(warmup):
        fn()
    times, errors = [], 0
    for _ in range(n):
        t0 = time.perf_counter()
        try:
            result = fn()
            elapsed = time.perf_counter() - t0
            if isinstance(result, str):
                d = json.loads(result)
                if not d.get("success"):
                    errors += 1
        except Exception:
            elapsed = time.perf_counter() - t0
            errors += 1
        times.append(elapsed)
    return times, errors


def _row(label: str, stats: Dict, col_w: int = 9) -> None:
    print(
        f"  {label:<46}  "
        f"{stats['mean_ms']:>{col_w}.2f}  "
        f"{stats['median_ms']:>{col_w}.2f}  "
        f"{stats['min_ms']:>{col_w}.2f}  "
        f"{stats['p95_ms']:>{col_w}.2f}  "
        f"{stats['max_ms']:>{col_w}.2f}  ms"
    )


# ---------------------------------------------------------------------------
# The "current main" approach — 3 separate _cdp_call() connections
# ---------------------------------------------------------------------------

def _baseline_cdp_click(endpoint: str, x: int, y: int, button: str = "left") -> str:
    """Replicate the previous 3-connection approach from the original PR."""
    from tools.browser_cdp_tool import _cdp_call, _run_async

    try:
        targets_result = _run_async(_cdp_call(endpoint, "Target.getTargets", {}, None, 10.0))
        page_target = None
        for t in targets_result.get("targetInfos", []):
            if t.get("type") == "page" and t.get("attached", True):
                page_target = t["targetId"]
                break
    except Exception:
        page_target = None

    mouse_params = {"type": "", "x": x, "y": y, "button": button, "clickCount": 1}
    try:
        _run_async(_cdp_call(endpoint, "Input.dispatchMouseEvent",
                             {**mouse_params, "type": "mousePressed"}, page_target, 10.0))
        _run_async(_cdp_call(endpoint, "Input.dispatchMouseEvent",
                             {**mouse_params, "type": "mouseReleased"}, page_target, 10.0))
    except Exception as e:
        return json.dumps({"success": False, "error": str(e)})
    return json.dumps({"success": True, "clicked_at": {"x": x, "y": y}, "method": "baseline"})


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

def run_benchmark(iterations: int = 300, warmup: int = 20) -> None:
    print(f"\n{'=' * 78}")
    print(f"  browser_click Coordinate Click: Current Main vs Optimized (1-conn)")
    print(f"  Real Lightpanda WS: {LIGHTPANDA_WS}")
    print(f"{'=' * 78}")
    print(f"  Iterations: {iterations}  |  Warmup: {warmup}")

    # pre-flight
    try:
        with urllib.request.urlopen("http://127.0.0.1:63372/json/version", timeout=2) as r:
            info = json.loads(r.read())
            assert "webSocketDebuggerUrl" in info
        print(f"  ✓ Lightpanda CDP: {info.get('webSocketDebuggerUrl')}")
    except Exception as e:
        print(f"  ✗ Lightpanda not reachable: {e}")
        return

    try:
        with urllib.request.urlopen(f"http://127.0.0.1:{AGENT_BROWSER_PORT}/api/sessions", timeout=2) as r:
            sessions = json.loads(r.read())
        print(f"  ✓ agent-browser: {len(sessions)} session(s)")
        ab_ok = True
    except Exception:
        print(f"  ⚠  agent-browser not reachable — ref-click IPC baseline skipped")
        ab_ok = False

    import importlib
    import tools.browser_tool as bt
    import tools.browser_cdp_tool as cdp_mod
    importlib.reload(cdp_mod)
    importlib.reload(bt)
    bt._is_camofox_mode = lambda: False
    _orig_resolve = cdp_mod._resolve_cdp_endpoint

    # -----------------------------------------------------------------------
    # 1. Baseline: current-main 3-connection approach
    # -----------------------------------------------------------------------
    print(f"\n  [1/3] Baseline (current main — 3 separate WS connections per click)")
    print(f"        Warmup {warmup}, then {iterations} iterations...")

    base_times, base_err = _bench(
        lambda: _baseline_cdp_click(LIGHTPANDA_WS, 150, 200),
        warmup, iterations,
    )
    base_stats = _stats(base_times)
    print(f"        Done — {base_err} errors, mean={base_stats['mean_ms']:.2f}ms")

    # -----------------------------------------------------------------------
    # 2. Optimized: single-connection (this PR)
    # -----------------------------------------------------------------------
    print(f"\n  [2/3] Optimized (this PR — 1 WS connection, pipelined mouse events)")
    print(f"        Warmup {warmup}, then {iterations} iterations...")

    cdp_mod._resolve_cdp_endpoint = lambda: LIGHTPANDA_WS
    opt_times, opt_err = _bench(
        lambda: bt.browser_click(x=150.0, y=200.0, task_id="bench"),
        warmup, iterations,
    )
    cdp_mod._resolve_cdp_endpoint = _orig_resolve
    opt_stats = _stats(opt_times)
    print(f"        Done — {opt_err} errors, mean={opt_stats['mean_ms']:.2f}ms")

    # -----------------------------------------------------------------------
    # 3. agent-browser HTTP IPC reference (what a ref click costs)
    # -----------------------------------------------------------------------
    if ab_ok:
        print(f"\n  [3/3] agent-browser HTTP IPC (reference for ref-click latency)")
        print(f"        Warmup {warmup}, then {iterations} iterations...")

        ab_times = []
        for _ in range(warmup):
            urllib.request.urlopen(f"http://127.0.0.1:{AGENT_BROWSER_PORT}/api/sessions", timeout=5).read()
        for _ in range(iterations):
            t0 = time.perf_counter()
            urllib.request.urlopen(f"http://127.0.0.1:{AGENT_BROWSER_PORT}/api/sessions", timeout=5).read()
            ab_times.append(time.perf_counter() - t0)
        ab_stats = _stats(ab_times)
        print(f"        Done — mean={ab_stats['mean_ms']:.2f}ms")

    # -----------------------------------------------------------------------
    # Results
    # -----------------------------------------------------------------------
    col_w = 9
    print(f"\n{'─' * 78}")
    print(f"  {'Approach':<46}  {'Mean':>{col_w}}  {'Median':>{col_w}}  {'Min':>{col_w}}  {'p95':>{col_w}}  {'Max':>{col_w}}")
    print(f"{'─' * 78}")
    _row("Baseline  (3 connections, sequential)", base_stats, col_w)
    _row("Optimized (1 connection, pipelined)  ", opt_stats,  col_w)
    if ab_ok:
        _row("Ref-click IPC baseline (1 HTTP req)  ", ab_stats,  col_w)
    print(f"{'─' * 78}")

    speedup = base_stats["mean_ms"] / opt_stats["mean_ms"]
    saved_ms = base_stats["mean_ms"] - opt_stats["mean_ms"]
    print(f"\n  Results:")
    print(f"    Speedup (mean):     {speedup:.2f}x  ({saved_ms:.2f} ms saved per click)")
    print(f"    Speedup (median):   {base_stats['median_ms'] / opt_stats['median_ms']:.2f}x")
    print(f"    Speedup (p95):      {base_stats['p95_ms'] / opt_stats['p95_ms']:.2f}x")

    if ab_ok:
        cdp_vs_ref = opt_stats["mean_ms"] / ab_stats["mean_ms"]
        print(f"\n    Optimized CDP vs ref-click: {cdp_vs_ref:.1f}x  "
              f"(+{opt_stats['mean_ms'] - ab_stats['mean_ms']:.2f} ms over ref)")
        print(f"    For scenarios where ref-click works, it's still faster.")
        print(f"    For cross-origin iframes/shadow DOM/canvas, coordinate click")
        print(f"    is the only option — {opt_stats['mean_ms']:.1f}ms is the cost.")

    print(f"\n  Why the optimized approach is faster:")
    print(f"    • Baseline: 3 sequential websocket.connect() calls")
    print(f"      Each = TCP handshake + WS upgrade + message + close")
    print(f"    • Optimized: 1 connect, then 4 messages in sequence")
    print(f"      mousePressed + mouseReleased are pipelined (sent before")
    print(f"      awaiting either response) — they travel in the same burst.")
    print(f"    • Savings come entirely from eliminating 2 WS handshakes.")
    print()


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--iterations", type=int, default=300)
    parser.add_argument("--warmup", type=int, default=20)
    args = parser.parse_args()
    run_benchmark(iterations=args.iterations, warmup=args.warmup)