feat: add compositor-level coordinate click to browser_click
Add optional x/y parameters to browser_click for viewport-coordinate clicking via CDP Input.dispatchMouseEvent. When coordinates are provided, clicks are dispatched at the browser compositor level — Chrome does its own hit-testing, bypassing DOM selectors entirely. Use cases where ref-based click fails but coordinate click works: - Cross-origin iframes (OOPIFs) - Closed shadow DOM - Canvas/WebGL elements - Dynamic overlays where the snapshot may be stale Implementation: - CDP path (preferred): Input.dispatchMouseEvent via WebSocket (Target.getTargets + mousePressed + mouseReleased) - agent-browser fallback: mouse move/down/up when no CDP endpoint available - ref is no longer required — either ref OR x+y must be provided Benchmark (real Lightpanda WS at ws://127.0.0.1:63372, 200 iterations): CDP coord click: 3.71ms mean (2.97ms median, 2.61ms min, 7.01ms p95) Single WS conn baseline: 1.57ms mean (cost per connection open+call) agent-browser IPC: 0.20ms mean per HTTP call The 3.71ms per CDP click comes from 3 sequential fresh WS connections (pre-existing architecture in browser_cdp_tool.py). A persistent WS connection pool would bring this to ~3.1ms (just the 2 mouse events). Both paths are well under the 100ms human perception threshold. Files: - tools/browser_tool.py: schema update (x/y, ref no longer required), _cdp_coordinate_click(), _coordinate_click_via_agent_browser(), updated browser_click() with validation and dispatch - tests/tools/test_browser_coordinate_click.py: 21 tests covering validation, CDP path, fallback path, ref preservation, schema, registry - scripts/benchmark_click_paths.py: real-browser latency benchmark
This commit is contained in:
kshitijk4poor
@@ -0,0 +1,190 @@
|
||||
"""
|
||||
Benchmark: CDP coordinate click (real Lightpanda) vs agent-browser IPC latency.
|
||||
|
||||
This measures:
|
||||
1. CDP path — browser_click(x, y) hitting real Lightpanda WS at :63372
|
||||
2. agent-browser HTTP IPC — raw HTTP request to /api/click (measures the IPC
|
||||
channel cost without needing a loaded page)
|
||||
|
||||
Usage: python scripts/benchmark_click_paths.py [--iterations N] [--warmup N]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from statistics import mean, median, stdev
|
||||
from typing import List, Dict, Tuple
|
||||
|
||||
sys.path.insert(0, "/private/tmp/hermes-coord-click")
|
||||
|
||||
LIGHTPANDA_WS = "ws://127.0.0.1:63372/"
|
||||
AGENT_BROWSER_BASE = "http://127.0.0.1:63371"
|
||||
|
||||
|
||||
def _stats(times_s: List[float]) -> Dict[str, float]:
|
||||
ms = [t * 1000 for t in times_s]
|
||||
return {
|
||||
"mean_ms": mean(ms),
|
||||
"median_ms": median(ms),
|
||||
"min_ms": min(ms),
|
||||
"max_ms": max(ms),
|
||||
"stdev_ms": stdev(ms) if len(ms) > 1 else 0.0,
|
||||
"p95_ms": sorted(ms)[int(len(ms) * 0.95)],
|
||||
}
|
||||
|
||||
|
||||
def _row(label: str, stats: Dict, col_w: int = 9) -> None:
|
||||
print(
|
||||
f" {label:<44} "
|
||||
f"{stats['mean_ms']:>{col_w}.2f} "
|
||||
f"{stats['median_ms']:>{col_w}.2f} "
|
||||
f"{stats['min_ms']:>{col_w}.2f} "
|
||||
f"{stats['p95_ms']:>{col_w}.2f} "
|
||||
f"{stats['max_ms']:>{col_w}.2f} ms"
|
||||
)
|
||||
|
||||
|
||||
def _bench(fn, n: int) -> Tuple[List[float], int]:
|
||||
times, errors = [], 0
|
||||
for _ in range(n):
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
result = fn()
|
||||
elapsed = time.perf_counter() - t0
|
||||
if isinstance(result, str):
|
||||
d = json.loads(result)
|
||||
if not d.get("success"):
|
||||
errors += 1
|
||||
except Exception:
|
||||
elapsed = time.perf_counter() - t0
|
||||
errors += 1
|
||||
times.append(elapsed)
|
||||
return times, errors
|
||||
|
||||
|
||||
def _http_get(url: str) -> float:
|
||||
"""Return elapsed seconds for a single HTTP GET (measures IPC round-trip)."""
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=5) as r:
|
||||
r.read()
|
||||
except Exception:
|
||||
pass
|
||||
return time.perf_counter() - t0
|
||||
|
||||
|
||||
def run_benchmark(iterations: int = 200, warmup: int = 15) -> None:
|
||||
print(f"\n{'=' * 76}")
|
||||
print(f" browser_click: CDP Coordinate Click (real Lightpanda) Benchmark")
|
||||
print(f"{'=' * 76}")
|
||||
print(f" Iterations: {iterations} | Warmup: {warmup}")
|
||||
|
||||
import importlib
|
||||
import tools.browser_tool as bt
|
||||
import tools.browser_cdp_tool as cdp_mod
|
||||
importlib.reload(cdp_mod)
|
||||
importlib.reload(bt)
|
||||
|
||||
bt._is_camofox_mode = lambda: False
|
||||
_orig_resolve = cdp_mod._resolve_cdp_endpoint
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# A. CDP coord click via real Lightpanda WS
|
||||
# -----------------------------------------------------------------------
|
||||
print(f"\n [A] CDP coord → Lightpanda WS ({LIGHTPANDA_WS})")
|
||||
cdp_mod._resolve_cdp_endpoint = lambda: LIGHTPANDA_WS
|
||||
|
||||
print(f" Warming up ({warmup})...")
|
||||
for _ in range(warmup):
|
||||
bt.browser_click(x=100.0, y=100.0, task_id="bench")
|
||||
|
||||
print(f" Benchmarking ({iterations})...")
|
||||
cdp_times, cdp_err = _bench(
|
||||
lambda: bt.browser_click(x=150.0, y=200.0, task_id="bench"),
|
||||
iterations,
|
||||
)
|
||||
cdp_mod._resolve_cdp_endpoint = _orig_resolve
|
||||
cdp_stats = _stats(cdp_times)
|
||||
print(f" Done — {cdp_err} errors, mean={cdp_stats['mean_ms']:.2f}ms")
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# B. agent-browser HTTP IPC latency (GET /api/sessions — lightweight ping)
|
||||
# -----------------------------------------------------------------------
|
||||
print(f"\n [B] agent-browser HTTP IPC round-trip (:63371/api/sessions)")
|
||||
print(f" Warming up ({warmup})...")
|
||||
for _ in range(warmup):
|
||||
_http_get(f"{AGENT_BROWSER_BASE}/api/sessions")
|
||||
|
||||
print(f" Benchmarking ({iterations})...")
|
||||
ab_times = []
|
||||
for _ in range(iterations):
|
||||
ab_times.append(_http_get(f"{AGENT_BROWSER_BASE}/api/sessions"))
|
||||
ab_stats = _stats(ab_times)
|
||||
print(f" Done — mean={ab_stats['mean_ms']:.2f}ms")
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# C. Raw Lightpanda WS latency (single CDP call — no click, just a ping)
|
||||
# This measures WS connection setup + 1 message round-trip
|
||||
# -----------------------------------------------------------------------
|
||||
print(f"\n [C] Raw single CDP call to Lightpanda (Target.getTargets baseline)")
|
||||
print(f" Warming up ({warmup})...")
|
||||
|
||||
async def _single_cdp():
|
||||
from tools.browser_cdp_tool import _cdp_call, _run_async
|
||||
return _run_async(_cdp_call(LIGHTPANDA_WS, "Target.getTargets", {}, None, 5.0))
|
||||
|
||||
def _time_single_cdp():
|
||||
from tools.browser_cdp_tool import _cdp_call, _run_async
|
||||
return _run_async(_cdp_call(LIGHTPANDA_WS, "Target.getTargets", {}, None, 5.0))
|
||||
|
||||
for _ in range(warmup):
|
||||
_time_single_cdp()
|
||||
|
||||
print(f" Benchmarking ({iterations})...")
|
||||
single_cdp_times = []
|
||||
for _ in range(iterations):
|
||||
t0 = time.perf_counter()
|
||||
_time_single_cdp()
|
||||
single_cdp_times.append(time.perf_counter() - t0)
|
||||
single_cdp_stats = _stats(single_cdp_times)
|
||||
print(f" Done — mean={single_cdp_stats['mean_ms']:.2f}ms per WS connection+call")
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Results
|
||||
# -----------------------------------------------------------------------
|
||||
col_w = 9
|
||||
print(f"\n{'─' * 76}")
|
||||
print(f" {'Path':<44} {'Mean':>{col_w}} {'Median':>{col_w}} {'Min':>{col_w}} {'p95':>{col_w}} {'Max':>{col_w}}")
|
||||
print(f"{'─' * 76}")
|
||||
_row("CDP coord (x,y) → Lightpanda [3 WS conns]", cdp_stats, col_w)
|
||||
_row("Single CDP call [1 WS conn baseline]", single_cdp_stats, col_w)
|
||||
_row("agent-browser HTTP IPC [1 request]", ab_stats, col_w)
|
||||
print(f"{'─' * 76}")
|
||||
|
||||
expected_3x = single_cdp_stats["mean_ms"] * 3
|
||||
print(f"\n Analysis:")
|
||||
print(f" • Full CDP click = {cdp_stats['mean_ms']:.2f}ms ({cdp_stats['mean_ms'] / single_cdp_stats['mean_ms']:.1f}× single call)")
|
||||
print(f" • Expected at 3 sequential WS connections: ~{expected_3x:.1f}ms")
|
||||
print(f" • Single WS conn+call baseline: {single_cdp_stats['mean_ms']:.2f}ms")
|
||||
print(f" • agent-browser HTTP IPC: {ab_stats['mean_ms']:.2f}ms per call")
|
||||
print(f" (ref click = ~1 IPC call, fallback mouse = ~3 IPC calls)")
|
||||
print(f" • Estimated ref click latency: ~{ab_stats['mean_ms']:.1f}ms")
|
||||
print(f" • Estimated fallback (mouse) latency: ~{ab_stats['mean_ms']*3:.1f}ms")
|
||||
print(f" • CDP coord vs ref estimate: {cdp_stats['mean_ms']:.1f}ms vs ~{ab_stats['mean_ms']:.1f}ms")
|
||||
cdp_vs_ref_est = cdp_stats["mean_ms"] / ab_stats["mean_ms"]
|
||||
print(f" Ratio: {cdp_vs_ref_est:.1f}x — overhead from 3 per-click WS conn setups.")
|
||||
print(f" • Both are well under 100ms human perception threshold.")
|
||||
print(f" • A persistent WS connection would reduce CDP clicks to ~{single_cdp_stats['mean_ms']*2:.1f}ms")
|
||||
print(f" (just mousePressed + mouseReleased on existing session).")
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--iterations", type=int, default=200)
|
||||
parser.add_argument("--warmup", type=int, default=15)
|
||||
args = parser.parse_args()
|
||||
run_benchmark(iterations=args.iterations, warmup=args.warmup)
|
||||
Reference in New Issue
Block a user