diff --git a/gateway/config.py b/gateway/config.py index ae5a0bac47..fc348e70d6 100644 --- a/gateway/config.py +++ b/gateway/config.py @@ -43,6 +43,7 @@ class Platform(Enum): HOMEASSISTANT = "homeassistant" EMAIL = "email" SMS = "sms" + DINGTALK = "dingtalk" @dataclass diff --git a/gateway/platforms/dingtalk.py b/gateway/platforms/dingtalk.py new file mode 100644 index 0000000000..38a8026418 --- /dev/null +++ b/gateway/platforms/dingtalk.py @@ -0,0 +1,340 @@ +""" +DingTalk platform adapter using Stream Mode. + +Uses dingtalk-stream SDK for real-time message reception without webhooks. +Responses are sent via DingTalk's session webhook (markdown format). + +Requires: + pip install dingtalk-stream httpx + DINGTALK_CLIENT_ID and DINGTALK_CLIENT_SECRET env vars + +Configuration in config.yaml: + platforms: + dingtalk: + enabled: true + extra: + client_id: "your-app-key" # or DINGTALK_CLIENT_ID env var + client_secret: "your-secret" # or DINGTALK_CLIENT_SECRET env var +""" + +import asyncio +import logging +import os +import time +import uuid +from datetime import datetime, timezone +from typing import Any, Dict, Optional + +try: + import dingtalk_stream + from dingtalk_stream import ChatbotHandler, ChatbotMessage + DINGTALK_STREAM_AVAILABLE = True +except ImportError: + DINGTALK_STREAM_AVAILABLE = False + dingtalk_stream = None # type: ignore[assignment] + +try: + import httpx + HTTPX_AVAILABLE = True +except ImportError: + HTTPX_AVAILABLE = False + httpx = None # type: ignore[assignment] + +from gateway.config import Platform, PlatformConfig +from gateway.platforms.base import ( + BasePlatformAdapter, + MessageEvent, + MessageType, + SendResult, +) + +logger = logging.getLogger(__name__) + +MAX_MESSAGE_LENGTH = 20000 +DEDUP_WINDOW_SECONDS = 300 +DEDUP_MAX_SIZE = 1000 +RECONNECT_BACKOFF = [2, 5, 10, 30, 60] + + +def check_dingtalk_requirements() -> bool: + """Check if DingTalk dependencies are available and configured.""" + if not DINGTALK_STREAM_AVAILABLE or not HTTPX_AVAILABLE: + return False + if not os.getenv("DINGTALK_CLIENT_ID") and not os.getenv("DINGTALK_CLIENT_SECRET"): + return False + return True + + +class DingTalkAdapter(BasePlatformAdapter): + """DingTalk chatbot adapter using Stream Mode. + + The dingtalk-stream SDK maintains a long-lived WebSocket connection. + Incoming messages arrive via a ChatbotHandler callback. Replies are + sent via the incoming message's session_webhook URL using httpx. + """ + + MAX_MESSAGE_LENGTH = MAX_MESSAGE_LENGTH + + def __init__(self, config: PlatformConfig): + super().__init__(config, Platform.DINGTALK) + + extra = config.extra or {} + self._client_id: str = extra.get("client_id") or os.getenv("DINGTALK_CLIENT_ID", "") + self._client_secret: str = extra.get("client_secret") or os.getenv("DINGTALK_CLIENT_SECRET", "") + + self._stream_client: Any = None + self._stream_task: Optional[asyncio.Task] = None + self._http_client: Optional["httpx.AsyncClient"] = None + + # Message deduplication: msg_id -> timestamp + self._seen_messages: Dict[str, float] = {} + # Map chat_id -> session_webhook for reply routing + self._session_webhooks: Dict[str, str] = {} + + # -- Connection lifecycle ----------------------------------------------- + + async def connect(self) -> bool: + """Connect to DingTalk via Stream Mode.""" + if not DINGTALK_STREAM_AVAILABLE: + logger.warning("[%s] dingtalk-stream not installed. Run: pip install dingtalk-stream", self.name) + return False + if not HTTPX_AVAILABLE: + logger.warning("[%s] httpx not installed. Run: pip install httpx", self.name) + return False + if not self._client_id or not self._client_secret: + logger.warning("[%s] DINGTALK_CLIENT_ID and DINGTALK_CLIENT_SECRET required", self.name) + return False + + try: + self._http_client = httpx.AsyncClient(timeout=30.0) + + credential = dingtalk_stream.Credential(self._client_id, self._client_secret) + self._stream_client = dingtalk_stream.DingTalkStreamClient(credential) + + # Capture the current event loop for cross-thread dispatch + loop = asyncio.get_running_loop() + handler = _IncomingHandler(self, loop) + self._stream_client.register_callback_handler( + dingtalk_stream.ChatbotMessage.TOPIC, handler + ) + + self._stream_task = asyncio.create_task(self._run_stream()) + self._mark_connected() + logger.info("[%s] Connected via Stream Mode", self.name) + return True + except Exception as e: + logger.error("[%s] Failed to connect: %s", self.name, e) + return False + + async def _run_stream(self) -> None: + """Run the blocking stream client with auto-reconnection.""" + backoff_idx = 0 + while self._running: + try: + logger.debug("[%s] Starting stream client...", self.name) + await asyncio.to_thread(self._stream_client.start) + except asyncio.CancelledError: + return + except Exception as e: + if not self._running: + return + logger.warning("[%s] Stream client error: %s", self.name, e) + + if not self._running: + return + + delay = RECONNECT_BACKOFF[min(backoff_idx, len(RECONNECT_BACKOFF) - 1)] + logger.info("[%s] Reconnecting in %ds...", self.name, delay) + await asyncio.sleep(delay) + backoff_idx += 1 + + async def disconnect(self) -> None: + """Disconnect from DingTalk.""" + self._running = False + self._mark_disconnected() + + if self._stream_task: + self._stream_task.cancel() + try: + await self._stream_task + except asyncio.CancelledError: + pass + self._stream_task = None + + if self._http_client: + await self._http_client.aclose() + self._http_client = None + + self._stream_client = None + self._session_webhooks.clear() + self._seen_messages.clear() + logger.info("[%s] Disconnected", self.name) + + # -- Inbound message processing ----------------------------------------- + + async def _on_message(self, message: "ChatbotMessage") -> None: + """Process an incoming DingTalk chatbot message.""" + msg_id = getattr(message, "message_id", None) or uuid.uuid4().hex + if self._is_duplicate(msg_id): + logger.debug("[%s] Duplicate message %s, skipping", self.name, msg_id) + return + + text = self._extract_text(message) + if not text: + logger.debug("[%s] Empty message, skipping", self.name) + return + + # Chat context + conversation_id = getattr(message, "conversation_id", "") or "" + conversation_type = getattr(message, "conversation_type", "1") + is_group = str(conversation_type) == "2" + sender_id = getattr(message, "sender_id", "") or "" + sender_nick = getattr(message, "sender_nick", "") or sender_id + sender_staff_id = getattr(message, "sender_staff_id", "") or "" + + chat_id = conversation_id or sender_id + chat_type = "group" if is_group else "dm" + + # Store session webhook for reply routing + session_webhook = getattr(message, "session_webhook", None) or "" + if session_webhook and chat_id: + self._session_webhooks[chat_id] = session_webhook + + source = self.build_source( + chat_id=chat_id, + chat_name=getattr(message, "conversation_title", None), + chat_type=chat_type, + user_id=sender_id, + user_name=sender_nick, + user_id_alt=sender_staff_id if sender_staff_id else None, + ) + + # Parse timestamp + create_at = getattr(message, "create_at", None) + try: + timestamp = datetime.fromtimestamp(int(create_at) / 1000, tz=timezone.utc) if create_at else datetime.now(tz=timezone.utc) + except (ValueError, OSError, TypeError): + timestamp = datetime.now(tz=timezone.utc) + + event = MessageEvent( + text=text, + message_type=MessageType.TEXT, + source=source, + message_id=msg_id, + raw_message=message, + timestamp=timestamp, + ) + + logger.debug("[%s] Message from %s in %s: %s", + self.name, sender_nick, chat_id[:20] if chat_id else "?", text[:50]) + await self.handle_message(event) + + @staticmethod + def _extract_text(message: "ChatbotMessage") -> str: + """Extract plain text from a DingTalk chatbot message.""" + text = getattr(message, "text", None) or "" + if isinstance(text, dict): + content = text.get("content", "").strip() + else: + content = str(text).strip() + + # Fall back to rich text if present + if not content: + rich_text = getattr(message, "rich_text", None) + if rich_text and isinstance(rich_text, list): + parts = [item["text"] for item in rich_text + if isinstance(item, dict) and item.get("text")] + content = " ".join(parts).strip() + return content + + # -- Deduplication ------------------------------------------------------ + + def _is_duplicate(self, msg_id: str) -> bool: + """Check and record a message ID. Returns True if already seen.""" + now = time.time() + if len(self._seen_messages) > DEDUP_MAX_SIZE: + cutoff = now - DEDUP_WINDOW_SECONDS + self._seen_messages = {k: v for k, v in self._seen_messages.items() if v > cutoff} + + if msg_id in self._seen_messages: + return True + self._seen_messages[msg_id] = now + return False + + # -- Outbound messaging ------------------------------------------------- + + async def send( + self, + chat_id: str, + content: str, + reply_to: Optional[str] = None, + metadata: Optional[Dict[str, Any]] = None, + ) -> SendResult: + """Send a markdown reply via DingTalk session webhook.""" + metadata = metadata or {} + + session_webhook = metadata.get("session_webhook") or self._session_webhooks.get(chat_id) + if not session_webhook: + return SendResult(success=False, + error="No session_webhook available. Reply must follow an incoming message.") + + if not self._http_client: + return SendResult(success=False, error="HTTP client not initialized") + + payload = { + "msgtype": "markdown", + "markdown": {"title": "Hermes", "text": content[:self.MAX_MESSAGE_LENGTH]}, + } + + try: + resp = await self._http_client.post(session_webhook, json=payload, timeout=15.0) + if resp.status_code < 300: + return SendResult(success=True, message_id=uuid.uuid4().hex[:12]) + body = resp.text + logger.warning("[%s] Send failed HTTP %d: %s", self.name, resp.status_code, body[:200]) + return SendResult(success=False, error=f"HTTP {resp.status_code}: {body[:200]}") + except httpx.TimeoutException: + return SendResult(success=False, error="Timeout sending message to DingTalk") + except Exception as e: + logger.error("[%s] Send error: %s", self.name, e) + return SendResult(success=False, error=str(e)) + + async def send_typing(self, chat_id: str, metadata=None) -> None: + """DingTalk does not support typing indicators.""" + pass + + async def get_chat_info(self, chat_id: str) -> Dict[str, Any]: + """Return basic info about a DingTalk conversation.""" + return {"name": chat_id, "type": "group" if "group" in chat_id.lower() else "dm"} + + +# --------------------------------------------------------------------------- +# Internal stream handler +# --------------------------------------------------------------------------- + +class _IncomingHandler(ChatbotHandler if DINGTALK_STREAM_AVAILABLE else object): + """dingtalk-stream ChatbotHandler that forwards messages to the adapter.""" + + def __init__(self, adapter: DingTalkAdapter, loop: asyncio.AbstractEventLoop): + if DINGTALK_STREAM_AVAILABLE: + super().__init__() + self._adapter = adapter + self._loop = loop + + def process(self, message: "ChatbotMessage"): + """Called by dingtalk-stream in its thread when a message arrives. + + Schedules the async handler on the main event loop. + """ + loop = self._loop + if loop is None or loop.is_closed(): + logger.error("[DingTalk] Event loop unavailable, cannot dispatch message") + return dingtalk_stream.AckMessage.STATUS_OK, "OK" + + future = asyncio.run_coroutine_threadsafe(self._adapter._on_message(message), loop) + try: + future.result(timeout=60) + except Exception: + logger.exception("[DingTalk] Error processing incoming message") + + return dingtalk_stream.AckMessage.STATUS_OK, "OK" diff --git a/gateway/run.py b/gateway/run.py index 93b7bd6c42..4d8910c225 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -1140,6 +1140,13 @@ class GatewayRunner: return None return SmsAdapter(config) + elif platform == Platform.DINGTALK: + from gateway.platforms.dingtalk import DingTalkAdapter, check_dingtalk_requirements + if not check_dingtalk_requirements(): + logger.warning("DingTalk: dingtalk-stream not installed or DINGTALK_CLIENT_ID/SECRET not set") + return None + return DingTalkAdapter(config) + return None def _is_user_authorized(self, source: SessionSource) -> bool: diff --git a/hermes_cli/config.py b/hermes_cli/config.py index 62d8a19a77..a72c0ff638 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -34,6 +34,7 @@ _EXTRA_ENV_KEYS = frozenset({ "DISCORD_HOME_CHANNEL", "TELEGRAM_HOME_CHANNEL", "SIGNAL_ACCOUNT", "SIGNAL_HTTP_URL", "SIGNAL_ALLOWED_USERS", "SIGNAL_GROUP_ALLOWED_USERS", + "DINGTALK_CLIENT_ID", "DINGTALK_CLIENT_SECRET", "TERMINAL_ENV", "TERMINAL_SSH_KEY", "TERMINAL_SSH_PORT", "WHATSAPP_MODE", "WHATSAPP_ENABLED", }) @@ -354,6 +355,11 @@ DEFAULT_CONFIG = { "tirith_path": "tirith", "tirith_timeout": 5, "tirith_fail_open": True, + "website_blocklist": { + "enabled": False, + "domains": [], + "shared_files": [], + }, }, # Config schema version - bump this when adding new required fields diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 32d90ac6a9..e6cb01a2ba 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -784,6 +784,7 @@ def cmd_model(args): "opencode-go": "OpenCode Go", "ai-gateway": "AI Gateway", "kilocode": "Kilo Code", + "alibaba": "Alibaba Cloud (DashScope)", "custom": "Custom endpoint", } active_label = provider_labels.get(active, active) @@ -807,6 +808,7 @@ def cmd_model(args): ("opencode-zen", "OpenCode Zen (35+ curated models, pay-as-you-go)"), ("opencode-go", "OpenCode Go (open models, $10/month subscription)"), ("ai-gateway", "AI Gateway (Vercel — 200+ models, pay-per-use)"), + ("alibaba", "Alibaba Cloud / DashScope (Qwen models, Anthropic-compatible)"), ] # Add user-defined custom providers from config.yaml @@ -875,7 +877,7 @@ def cmd_model(args): _model_flow_anthropic(config, current_model) elif selected_provider == "kimi-coding": _model_flow_kimi(config, current_model) - elif selected_provider in ("zai", "minimax", "minimax-cn", "kilocode", "opencode-zen", "opencode-go", "ai-gateway"): + elif selected_provider in ("zai", "minimax", "minimax-cn", "kilocode", "opencode-zen", "opencode-go", "ai-gateway", "alibaba"): _model_flow_api_key_provider(config, selected_provider, current_model) diff --git a/hermes_cli/setup.py b/hermes_cli/setup.py index a79844def5..b635377f53 100644 --- a/hermes_cli/setup.py +++ b/hermes_cli/setup.py @@ -738,6 +738,7 @@ def setup_model_provider(config: dict): "Kilo Code (Kilo Gateway API)", "Anthropic (Claude models — API key or Claude Code subscription)", "AI Gateway (Vercel — 200+ models, pay-per-use)", + "Alibaba Cloud / DashScope (Qwen models via Anthropic-compatible API)", "OpenCode Zen (35+ curated models, pay-as-you-go)", "OpenCode Go (open models, $10/month subscription)", ] @@ -1313,7 +1314,39 @@ def setup_model_provider(config: dict): _update_config_for_provider("ai-gateway", pconfig.inference_base_url, default_model="anthropic/claude-opus-4.6") _set_model_provider(config, "ai-gateway", pconfig.inference_base_url) - elif provider_idx == 11: # OpenCode Zen + elif provider_idx == 11: # Alibaba Cloud / DashScope + selected_provider = "alibaba" + print() + print_header("Alibaba Cloud / DashScope API Key") + pconfig = PROVIDER_REGISTRY["alibaba"] + print_info(f"Provider: {pconfig.name}") + print_info("Get your API key at: https://modelstudio.console.alibabacloud.com/") + print() + + existing_key = get_env_value("DASHSCOPE_API_KEY") + if existing_key: + print_info(f"Current: {existing_key[:8]}... (configured)") + if prompt_yes_no("Update API key?", False): + new_key = prompt(" DashScope API key", password=True) + if new_key: + save_env_value("DASHSCOPE_API_KEY", new_key) + print_success("DashScope API key updated") + else: + new_key = prompt(" DashScope API key", password=True) + if new_key: + save_env_value("DASHSCOPE_API_KEY", new_key) + print_success("DashScope API key saved") + else: + print_warning("Skipped - agent won't work without an API key") + + # Clear custom endpoint vars if switching + if existing_custom: + save_env_value("OPENAI_BASE_URL", "") + save_env_value("OPENAI_API_KEY", "") + _update_config_for_provider("alibaba", pconfig.inference_base_url, default_model="qwen3.5-plus") + _set_model_provider(config, "alibaba", pconfig.inference_base_url) + + elif provider_idx == 12: # OpenCode Zen selected_provider = "opencode-zen" print() print_header("OpenCode Zen API Key") @@ -1346,7 +1379,7 @@ def setup_model_provider(config: dict): _set_model_provider(config, "opencode-zen", pconfig.inference_base_url) selected_base_url = pconfig.inference_base_url - elif provider_idx == 12: # OpenCode Go + elif provider_idx == 13: # OpenCode Go selected_provider = "opencode-go" print() print_header("OpenCode Go API Key") @@ -1379,7 +1412,7 @@ def setup_model_provider(config: dict): _set_model_provider(config, "opencode-go", pconfig.inference_base_url) selected_base_url = pconfig.inference_base_url - # else: provider_idx == 13 (Keep current) — only shown when a provider already exists + # else: provider_idx == 14 (Keep current) — only shown when a provider already exists # Normalize "keep current" to an explicit provider so downstream logic # doesn't fall back to the generic OpenRouter/static-model path. if selected_provider is None: diff --git a/hermes_cli/tools_config.py b/hermes_cli/tools_config.py index d106d0c472..c15c069e5d 100644 --- a/hermes_cli/tools_config.py +++ b/hermes_cli/tools_config.py @@ -110,6 +110,7 @@ PLATFORMS = { "whatsapp": {"label": "📱 WhatsApp", "default_toolset": "hermes-whatsapp"}, "signal": {"label": "📡 Signal", "default_toolset": "hermes-signal"}, "email": {"label": "📧 Email", "default_toolset": "hermes-email"}, + "dingtalk": {"label": "💬 DingTalk", "default_toolset": "hermes-dingtalk"}, } diff --git a/skills/inference-sh/DESCRIPTION.md b/skills/inference-sh/DESCRIPTION.md new file mode 100644 index 0000000000..011ede4c16 --- /dev/null +++ b/skills/inference-sh/DESCRIPTION.md @@ -0,0 +1,19 @@ +# inference.sh + +Run 150+ AI applications in the cloud via the [inference.sh](https://inference.sh) platform. + +**One API key for everything** — access image generation, video creation, LLMs, search, 3D, and more through a single account. No need to manage separate API keys for each provider. + +## Available Skills + +- **cli**: Use the inference.sh CLI (`infsh`) via the terminal tool + +## What's Included + +- **Image Generation**: FLUX, Reve, Seedream, Grok Imagine, Gemini +- **Video Generation**: Veo, Wan, Seedance, OmniHuman, HunyuanVideo +- **LLMs**: Claude, Gemini, Kimi, GLM-4 (via OpenRouter) +- **Search**: Tavily, Exa +- **3D**: Rodin +- **Social**: Twitter/X automation +- **Audio**: TTS, voice cloning diff --git a/skills/inference-sh/cli/SKILL.md b/skills/inference-sh/cli/SKILL.md new file mode 100644 index 0000000000..79183f61c2 --- /dev/null +++ b/skills/inference-sh/cli/SKILL.md @@ -0,0 +1,155 @@ +--- +name: inference-sh-cli +description: "Run 150+ AI apps via inference.sh CLI (infsh) — image generation, video creation, LLMs, search, 3D, social automation. Uses the terminal tool. Triggers: inference.sh, infsh, ai apps, flux, veo, image generation, video generation, seedream, seedance, tavily" +version: 1.0.0 +author: okaris +license: MIT +metadata: + hermes: + tags: [AI, image-generation, video, LLM, search, inference, FLUX, Veo, Claude] + related_skills: [] +--- + +# inference.sh CLI + +Run 150+ AI apps in the cloud with a simple CLI. No GPU required. + +All commands use the **terminal tool** to run `infsh` commands. + +## When to Use + +- User asks to generate images (FLUX, Reve, Seedream, Grok, Gemini image) +- User asks to generate video (Veo, Wan, Seedance, OmniHuman) +- User asks about inference.sh or infsh +- User wants to run AI apps without managing individual provider APIs +- User asks for AI-powered search (Tavily, Exa) +- User needs avatar/lipsync generation + +## Prerequisites + +The `infsh` CLI must be installed and authenticated. Check with: + +```bash +infsh me +``` + +If not installed: + +```bash +curl -fsSL https://cli.inference.sh | sh +infsh login +``` + +See `references/authentication.md` for full setup details. + +## Workflow + +### 1. Always Search First + +Never guess app names — always search to find the correct app ID: + +```bash +infsh app list --search flux +infsh app list --search video +infsh app list --search image +``` + +### 2. Run an App + +Use the exact app ID from the search results. Always use `--json` for machine-readable output: + +```bash +infsh app run --input '{"prompt": "your prompt here"}' --json +``` + +### 3. Parse the Output + +The JSON output contains URLs to generated media. Present these to the user with `MEDIA:` for inline display. + +## Common Commands + +### Image Generation + +```bash +# Search for image apps +infsh app list --search image + +# FLUX Dev with LoRA +infsh app run falai/flux-dev-lora --input '{"prompt": "sunset over mountains", "num_images": 1}' --json + +# Gemini image generation +infsh app run google/gemini-2-5-flash-image --input '{"prompt": "futuristic city", "num_images": 1}' --json + +# Seedream (ByteDance) +infsh app run bytedance/seedream-5-lite --input '{"prompt": "nature scene"}' --json + +# Grok Imagine (xAI) +infsh app run xai/grok-imagine-image --input '{"prompt": "abstract art"}' --json +``` + +### Video Generation + +```bash +# Search for video apps +infsh app list --search video + +# Veo 3.1 (Google) +infsh app run google/veo-3-1-fast --input '{"prompt": "drone shot of coastline"}' --json + +# Seedance (ByteDance) +infsh app run bytedance/seedance-1-5-pro --input '{"prompt": "dancing figure", "resolution": "1080p"}' --json + +# Wan 2.5 +infsh app run falai/wan-2-5 --input '{"prompt": "person walking through city"}' --json +``` + +### Local File Uploads + +The CLI automatically uploads local files when you provide a path: + +```bash +# Upscale a local image +infsh app run falai/topaz-image-upscaler --input '{"image": "/path/to/photo.jpg", "upscale_factor": 2}' --json + +# Image-to-video from local file +infsh app run falai/wan-2-5-i2v --input '{"image": "/path/to/image.png", "prompt": "make it move"}' --json + +# Avatar with audio +infsh app run bytedance/omnihuman-1-5 --input '{"audio": "/path/to/audio.mp3", "image": "/path/to/face.jpg"}' --json +``` + +### Search & Research + +```bash +infsh app list --search search +infsh app run tavily/tavily-search --input '{"query": "latest AI news"}' --json +infsh app run exa/exa-search --input '{"query": "machine learning papers"}' --json +``` + +### Other Categories + +```bash +# 3D generation +infsh app list --search 3d + +# Audio / TTS +infsh app list --search tts + +# Twitter/X automation +infsh app list --search twitter +``` + +## Pitfalls + +1. **Never guess app IDs** — always run `infsh app list --search ` first. App IDs change and new apps are added frequently. +2. **Always use `--json`** — raw output is hard to parse. The `--json` flag gives structured output with URLs. +3. **Check authentication** — if commands fail with auth errors, run `infsh login` or verify `INFSH_API_KEY` is set. +4. **Long-running apps** — video generation can take 30-120 seconds. The terminal tool timeout should be sufficient, but warn the user it may take a moment. +5. **Input format** — the `--input` flag takes a JSON string. Make sure to properly escape quotes. + +## Reference Docs + +- `references/authentication.md` — Setup, login, API keys +- `references/app-discovery.md` — Searching and browsing the app catalog +- `references/running-apps.md` — Running apps, input formats, output handling +- `references/cli-reference.md` — Complete CLI command reference diff --git a/skills/inference-sh/cli/references/app-discovery.md b/skills/inference-sh/cli/references/app-discovery.md new file mode 100644 index 0000000000..adcac8c5dc --- /dev/null +++ b/skills/inference-sh/cli/references/app-discovery.md @@ -0,0 +1,112 @@ +# Discovering Apps + +## List All Apps + +```bash +infsh app list +``` + +## Pagination + +```bash +infsh app list --page 2 +``` + +## Filter by Category + +```bash +infsh app list --category image +infsh app list --category video +infsh app list --category audio +infsh app list --category text +infsh app list --category other +``` + +## Search + +```bash +infsh app search "flux" +infsh app search "video generation" +infsh app search "tts" -l +infsh app search "image" --category image +``` + +Or use the flag form: + +```bash +infsh app list --search "flux" +infsh app list --search "video generation" +infsh app list --search "tts" +``` + +## Featured Apps + +```bash +infsh app list --featured +``` + +## Newest First + +```bash +infsh app list --new +``` + +## Detailed View + +```bash +infsh app list -l +``` + +Shows table with app name, category, description, and featured status. + +## Save to File + +```bash +infsh app list --save apps.json +``` + +## Your Apps + +List apps you've deployed: + +```bash +infsh app my +infsh app my -l # detailed +``` + +## Get App Details + +```bash +infsh app get falai/flux-dev-lora +infsh app get falai/flux-dev-lora --json +``` + +Shows full app info including input/output schema. + +## Popular Apps by Category + +### Image Generation +- `falai/flux-dev-lora` - FLUX.2 Dev (high quality) +- `falai/flux-2-klein-lora` - FLUX.2 Klein (fastest) +- `infsh/sdxl` - Stable Diffusion XL +- `google/gemini-3-pro-image-preview` - Gemini 3 Pro +- `xai/grok-imagine-image` - Grok image generation + +### Video Generation +- `google/veo-3-1-fast` - Veo 3.1 Fast +- `google/veo-3` - Veo 3 +- `bytedance/seedance-1-5-pro` - Seedance 1.5 Pro +- `infsh/ltx-video-2` - LTX Video 2 (with audio) +- `bytedance/omnihuman-1-5` - OmniHuman avatar + +### Audio +- `infsh/dia-tts` - Conversational TTS +- `infsh/kokoro-tts` - Kokoro TTS +- `infsh/fast-whisper-large-v3` - Fast transcription +- `infsh/diffrythm` - Music generation + +## Documentation + +- [Browsing the Grid](https://inference.sh/docs/apps/browsing-grid) - Visual app browsing +- [Apps Overview](https://inference.sh/docs/apps/overview) - Understanding apps +- [Running Apps](https://inference.sh/docs/apps/running) - How to run apps diff --git a/skills/inference-sh/cli/references/authentication.md b/skills/inference-sh/cli/references/authentication.md new file mode 100644 index 0000000000..3b6519d3de --- /dev/null +++ b/skills/inference-sh/cli/references/authentication.md @@ -0,0 +1,59 @@ +# Authentication & Setup + +## Install the CLI + +```bash +curl -fsSL https://cli.inference.sh | sh +``` + +## Login + +```bash +infsh login +``` + +This opens a browser for authentication. After login, credentials are stored locally. + +## Check Authentication + +```bash +infsh me +``` + +Shows your user info if authenticated. + +## Environment Variable + +For CI/CD or scripts, set your API key: + +```bash +export INFSH_API_KEY=your-api-key +``` + +The environment variable overrides the config file. + +## Update CLI + +```bash +infsh update +``` + +Or reinstall: + +```bash +curl -fsSL https://cli.inference.sh | sh +``` + +## Troubleshooting + +| Error | Solution | +|-------|----------| +| "not authenticated" | Run `infsh login` | +| "command not found" | Reinstall CLI or add to PATH | +| "API key invalid" | Check `INFSH_API_KEY` or re-login | + +## Documentation + +- [CLI Setup](https://inference.sh/docs/extend/cli-setup) - Complete CLI installation guide +- [API Authentication](https://inference.sh/docs/api/authentication) - API key management +- [Secrets](https://inference.sh/docs/secrets/overview) - Managing credentials diff --git a/skills/inference-sh/cli/references/cli-reference.md b/skills/inference-sh/cli/references/cli-reference.md new file mode 100644 index 0000000000..50825825f7 --- /dev/null +++ b/skills/inference-sh/cli/references/cli-reference.md @@ -0,0 +1,104 @@ +# CLI Reference + +## Installation + +```bash +curl -fsSL https://cli.inference.sh | sh +``` + +## Global Commands + +| Command | Description | +|---------|-------------| +| `infsh help` | Show help | +| `infsh version` | Show CLI version | +| `infsh update` | Update CLI to latest | +| `infsh login` | Authenticate | +| `infsh me` | Show current user | + +## App Commands + +### Discovery + +| Command | Description | +|---------|-------------| +| `infsh app list` | List available apps | +| `infsh app list --category ` | Filter by category (image, video, audio, text, other) | +| `infsh app search ` | Search apps | +| `infsh app list --search ` | Search apps (flag form) | +| `infsh app list --featured` | Show featured apps | +| `infsh app list --new` | Sort by newest | +| `infsh app list --page ` | Pagination | +| `infsh app list -l` | Detailed table view | +| `infsh app list --save ` | Save to JSON file | +| `infsh app my` | List your deployed apps | +| `infsh app get ` | Get app details | +| `infsh app get --json` | Get app details as JSON | + +### Execution + +| Command | Description | +|---------|-------------| +| `infsh app run --input ` | Run app with input file | +| `infsh app run --input ''` | Run with inline JSON | +| `infsh app run --input --no-wait` | Run without waiting for completion | +| `infsh app sample ` | Show sample input | +| `infsh app sample --save ` | Save sample to file | + +## Task Commands + +| Command | Description | +|---------|-------------| +| `infsh task get ` | Get task status and result | +| `infsh task get --json` | Get task as JSON | +| `infsh task get --save ` | Save task result to file | + +### Development + +| Command | Description | +|---------|-------------| +| `infsh app init` | Create new app (interactive) | +| `infsh app init ` | Create new app with name | +| `infsh app test --input ` | Test app locally | +| `infsh app deploy` | Deploy app | +| `infsh app deploy --dry-run` | Validate without deploying | +| `infsh app pull ` | Pull app source | +| `infsh app pull --all` | Pull all your apps | + +## Environment Variables + +| Variable | Description | +|----------|-------------| +| `INFSH_API_KEY` | API key (overrides config) | + +## Shell Completions + +```bash +# Bash +infsh completion bash > /etc/bash_completion.d/infsh + +# Zsh +infsh completion zsh > "${fpath[1]}/_infsh" + +# Fish +infsh completion fish > ~/.config/fish/completions/infsh.fish +``` + +## App Name Format + +Apps use the format `namespace/app-name`: + +- `falai/flux-dev-lora` - fal.ai's FLUX 2 Dev +- `google/veo-3` - Google's Veo 3 +- `infsh/sdxl` - inference.sh's SDXL +- `bytedance/seedance-1-5-pro` - ByteDance's Seedance +- `xai/grok-imagine-image` - xAI's Grok + +Version pinning: `namespace/app-name@version` + +## Documentation + +- [CLI Setup](https://inference.sh/docs/extend/cli-setup) - Complete CLI installation guide +- [Running Apps](https://inference.sh/docs/apps/running) - How to run apps via CLI +- [Creating an App](https://inference.sh/docs/extend/creating-app) - Build your own apps +- [Deploying](https://inference.sh/docs/extend/deploying) - Deploy apps to the cloud diff --git a/skills/inference-sh/cli/references/running-apps.md b/skills/inference-sh/cli/references/running-apps.md new file mode 100644 index 0000000000..e930d5cfbc --- /dev/null +++ b/skills/inference-sh/cli/references/running-apps.md @@ -0,0 +1,171 @@ +# Running Apps + +## Basic Run + +```bash +infsh app run user/app-name --input input.json +``` + +## Inline JSON + +```bash +infsh app run falai/flux-dev-lora --input '{"prompt": "a sunset over mountains"}' +``` + +## Version Pinning + +```bash +infsh app run user/app-name@1.0.0 --input input.json +``` + +## Local File Uploads + +The CLI automatically uploads local files when you provide a file path instead of a URL. Any field that accepts a URL also accepts a local path: + +```bash +# Upscale a local image +infsh app run falai/topaz-image-upscaler --input '{"image": "/path/to/photo.jpg", "upscale_factor": 2}' + +# Image-to-video from local file +infsh app run falai/wan-2-5-i2v --input '{"image": "./my-image.png", "prompt": "make it move"}' + +# Avatar with local audio and image +infsh app run bytedance/omnihuman-1-5 --input '{"audio": "/path/to/speech.mp3", "image": "/path/to/face.jpg"}' + +# Post tweet with local media +infsh app run x/post-create --input '{"text": "Check this out!", "media": "./screenshot.png"}' +``` + +Supported paths: +- Absolute paths: `/home/user/images/photo.jpg` +- Relative paths: `./image.png`, `../data/video.mp4` +- Home directory: `~/Pictures/photo.jpg` + +## Generate Sample Input + +Before running, generate a sample input file: + +```bash +infsh app sample falai/flux-dev-lora +``` + +Save to file: + +```bash +infsh app sample falai/flux-dev-lora --save input.json +``` + +Then edit `input.json` and run: + +```bash +infsh app run falai/flux-dev-lora --input input.json +``` + +## Workflow Example + +### Image Generation with FLUX + +```bash +# 1. Get app details +infsh app get falai/flux-dev-lora + +# 2. Generate sample input +infsh app sample falai/flux-dev-lora --save input.json + +# 3. Edit input.json +# { +# "prompt": "a cat astronaut floating in space", +# "num_images": 1, +# "image_size": "landscape_16_9" +# } + +# 4. Run +infsh app run falai/flux-dev-lora --input input.json +``` + +### Video Generation with Veo + +```bash +# 1. Generate sample +infsh app sample google/veo-3-1-fast --save input.json + +# 2. Edit prompt +# { +# "prompt": "A drone shot flying over a forest at sunset" +# } + +# 3. Run +infsh app run google/veo-3-1-fast --input input.json +``` + +### Text-to-Speech + +```bash +# Quick inline run +infsh app run falai/kokoro-tts --input '{"text": "Hello, this is a test."}' +``` + +## Task Tracking + +When you run an app, the CLI shows the task ID: + +``` +Running falai/flux-dev-lora +Task ID: abc123def456 +``` + +For long-running tasks, you can check status anytime: + +```bash +# Check task status +infsh task get abc123def456 + +# Get result as JSON +infsh task get abc123def456 --json + +# Save result to file +infsh task get abc123def456 --save result.json +``` + +### Run Without Waiting + +For very long tasks, run in background: + +```bash +# Submit and return immediately +infsh app run google/veo-3 --input input.json --no-wait + +# Check later +infsh task get +``` + +## Output + +The CLI returns the app output directly. For file outputs (images, videos, audio), you'll receive URLs to download. + +Example output: + +```json +{ + "images": [ + { + "url": "https://cloud.inference.sh/...", + "content_type": "image/png" + } + ] +} +``` + +## Error Handling + +| Error | Cause | Solution | +|-------|-------|----------| +| "invalid input" | Schema mismatch | Check `infsh app get` for required fields | +| "app not found" | Wrong app name | Check `infsh app list --search` | +| "quota exceeded" | Out of credits | Check account balance | + +## Documentation + +- [Running Apps](https://inference.sh/docs/apps/running) - Complete running apps guide +- [Streaming Results](https://inference.sh/docs/api/sdk/streaming) - Real-time progress updates +- [Setup Parameters](https://inference.sh/docs/apps/setup-parameters) - Configuring app inputs diff --git a/tests/gateway/test_dingtalk.py b/tests/gateway/test_dingtalk.py new file mode 100644 index 0000000000..5c73253fbf --- /dev/null +++ b/tests/gateway/test_dingtalk.py @@ -0,0 +1,274 @@ +"""Tests for DingTalk platform adapter.""" +import asyncio +import json +from datetime import datetime, timezone +from unittest.mock import AsyncMock, MagicMock, patch, PropertyMock + +import pytest + +from gateway.config import Platform, PlatformConfig + + +# --------------------------------------------------------------------------- +# Requirements check +# --------------------------------------------------------------------------- + + +class TestDingTalkRequirements: + + def test_returns_false_when_sdk_missing(self, monkeypatch): + with patch.dict("sys.modules", {"dingtalk_stream": None}): + monkeypatch.setattr( + "gateway.platforms.dingtalk.DINGTALK_STREAM_AVAILABLE", False + ) + from gateway.platforms.dingtalk import check_dingtalk_requirements + assert check_dingtalk_requirements() is False + + def test_returns_false_when_env_vars_missing(self, monkeypatch): + monkeypatch.setattr( + "gateway.platforms.dingtalk.DINGTALK_STREAM_AVAILABLE", True + ) + monkeypatch.setattr("gateway.platforms.dingtalk.HTTPX_AVAILABLE", True) + monkeypatch.delenv("DINGTALK_CLIENT_ID", raising=False) + monkeypatch.delenv("DINGTALK_CLIENT_SECRET", raising=False) + from gateway.platforms.dingtalk import check_dingtalk_requirements + assert check_dingtalk_requirements() is False + + def test_returns_true_when_all_available(self, monkeypatch): + monkeypatch.setattr( + "gateway.platforms.dingtalk.DINGTALK_STREAM_AVAILABLE", True + ) + monkeypatch.setattr("gateway.platforms.dingtalk.HTTPX_AVAILABLE", True) + monkeypatch.setenv("DINGTALK_CLIENT_ID", "test-id") + monkeypatch.setenv("DINGTALK_CLIENT_SECRET", "test-secret") + from gateway.platforms.dingtalk import check_dingtalk_requirements + assert check_dingtalk_requirements() is True + + +# --------------------------------------------------------------------------- +# Adapter construction +# --------------------------------------------------------------------------- + + +class TestDingTalkAdapterInit: + + def test_reads_config_from_extra(self): + from gateway.platforms.dingtalk import DingTalkAdapter + config = PlatformConfig( + enabled=True, + extra={"client_id": "cfg-id", "client_secret": "cfg-secret"}, + ) + adapter = DingTalkAdapter(config) + assert adapter._client_id == "cfg-id" + assert adapter._client_secret == "cfg-secret" + assert adapter.name == "Dingtalk" # base class uses .title() + + def test_falls_back_to_env_vars(self, monkeypatch): + monkeypatch.setenv("DINGTALK_CLIENT_ID", "env-id") + monkeypatch.setenv("DINGTALK_CLIENT_SECRET", "env-secret") + from gateway.platforms.dingtalk import DingTalkAdapter + config = PlatformConfig(enabled=True) + adapter = DingTalkAdapter(config) + assert adapter._client_id == "env-id" + assert adapter._client_secret == "env-secret" + + +# --------------------------------------------------------------------------- +# Message text extraction +# --------------------------------------------------------------------------- + + +class TestExtractText: + + def test_extracts_dict_text(self): + from gateway.platforms.dingtalk import DingTalkAdapter + msg = MagicMock() + msg.text = {"content": " hello world "} + msg.rich_text = None + assert DingTalkAdapter._extract_text(msg) == "hello world" + + def test_extracts_string_text(self): + from gateway.platforms.dingtalk import DingTalkAdapter + msg = MagicMock() + msg.text = "plain text" + msg.rich_text = None + assert DingTalkAdapter._extract_text(msg) == "plain text" + + def test_falls_back_to_rich_text(self): + from gateway.platforms.dingtalk import DingTalkAdapter + msg = MagicMock() + msg.text = "" + msg.rich_text = [{"text": "part1"}, {"text": "part2"}, {"image": "url"}] + assert DingTalkAdapter._extract_text(msg) == "part1 part2" + + def test_returns_empty_for_no_content(self): + from gateway.platforms.dingtalk import DingTalkAdapter + msg = MagicMock() + msg.text = "" + msg.rich_text = None + assert DingTalkAdapter._extract_text(msg) == "" + + +# --------------------------------------------------------------------------- +# Deduplication +# --------------------------------------------------------------------------- + + +class TestDeduplication: + + def test_first_message_not_duplicate(self): + from gateway.platforms.dingtalk import DingTalkAdapter + adapter = DingTalkAdapter(PlatformConfig(enabled=True)) + assert adapter._is_duplicate("msg-1") is False + + def test_second_same_message_is_duplicate(self): + from gateway.platforms.dingtalk import DingTalkAdapter + adapter = DingTalkAdapter(PlatformConfig(enabled=True)) + adapter._is_duplicate("msg-1") + assert adapter._is_duplicate("msg-1") is True + + def test_different_messages_not_duplicate(self): + from gateway.platforms.dingtalk import DingTalkAdapter + adapter = DingTalkAdapter(PlatformConfig(enabled=True)) + adapter._is_duplicate("msg-1") + assert adapter._is_duplicate("msg-2") is False + + def test_cache_cleanup_on_overflow(self): + from gateway.platforms.dingtalk import DingTalkAdapter, DEDUP_MAX_SIZE + adapter = DingTalkAdapter(PlatformConfig(enabled=True)) + # Fill beyond max + for i in range(DEDUP_MAX_SIZE + 10): + adapter._is_duplicate(f"msg-{i}") + # Cache should have been pruned + assert len(adapter._seen_messages) <= DEDUP_MAX_SIZE + 10 + + +# --------------------------------------------------------------------------- +# Send +# --------------------------------------------------------------------------- + + +class TestSend: + + @pytest.mark.asyncio + async def test_send_posts_to_webhook(self): + from gateway.platforms.dingtalk import DingTalkAdapter + adapter = DingTalkAdapter(PlatformConfig(enabled=True)) + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = "OK" + + mock_client = AsyncMock() + mock_client.post = AsyncMock(return_value=mock_response) + adapter._http_client = mock_client + + result = await adapter.send( + "chat-123", "Hello!", + metadata={"session_webhook": "https://dingtalk.example/webhook"} + ) + assert result.success is True + mock_client.post.assert_called_once() + call_args = mock_client.post.call_args + assert call_args[0][0] == "https://dingtalk.example/webhook" + payload = call_args[1]["json"] + assert payload["msgtype"] == "markdown" + assert payload["markdown"]["title"] == "Hermes" + assert payload["markdown"]["text"] == "Hello!" + + @pytest.mark.asyncio + async def test_send_fails_without_webhook(self): + from gateway.platforms.dingtalk import DingTalkAdapter + adapter = DingTalkAdapter(PlatformConfig(enabled=True)) + adapter._http_client = AsyncMock() + + result = await adapter.send("chat-123", "Hello!") + assert result.success is False + assert "session_webhook" in result.error + + @pytest.mark.asyncio + async def test_send_uses_cached_webhook(self): + from gateway.platforms.dingtalk import DingTalkAdapter + adapter = DingTalkAdapter(PlatformConfig(enabled=True)) + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_client = AsyncMock() + mock_client.post = AsyncMock(return_value=mock_response) + adapter._http_client = mock_client + adapter._session_webhooks["chat-123"] = "https://cached.example/webhook" + + result = await adapter.send("chat-123", "Hello!") + assert result.success is True + assert mock_client.post.call_args[0][0] == "https://cached.example/webhook" + + @pytest.mark.asyncio + async def test_send_handles_http_error(self): + from gateway.platforms.dingtalk import DingTalkAdapter + adapter = DingTalkAdapter(PlatformConfig(enabled=True)) + + mock_response = MagicMock() + mock_response.status_code = 400 + mock_response.text = "Bad Request" + mock_client = AsyncMock() + mock_client.post = AsyncMock(return_value=mock_response) + adapter._http_client = mock_client + + result = await adapter.send( + "chat-123", "Hello!", + metadata={"session_webhook": "https://example/webhook"} + ) + assert result.success is False + assert "400" in result.error + + +# --------------------------------------------------------------------------- +# Connect / disconnect +# --------------------------------------------------------------------------- + + +class TestConnect: + + @pytest.mark.asyncio + async def test_connect_fails_without_sdk(self, monkeypatch): + monkeypatch.setattr( + "gateway.platforms.dingtalk.DINGTALK_STREAM_AVAILABLE", False + ) + from gateway.platforms.dingtalk import DingTalkAdapter + adapter = DingTalkAdapter(PlatformConfig(enabled=True)) + result = await adapter.connect() + assert result is False + + @pytest.mark.asyncio + async def test_connect_fails_without_credentials(self): + from gateway.platforms.dingtalk import DingTalkAdapter + adapter = DingTalkAdapter(PlatformConfig(enabled=True)) + adapter._client_id = "" + adapter._client_secret = "" + result = await adapter.connect() + assert result is False + + @pytest.mark.asyncio + async def test_disconnect_cleans_up(self): + from gateway.platforms.dingtalk import DingTalkAdapter + adapter = DingTalkAdapter(PlatformConfig(enabled=True)) + adapter._session_webhooks["a"] = "http://x" + adapter._seen_messages["b"] = 1.0 + adapter._http_client = AsyncMock() + adapter._stream_task = None + + await adapter.disconnect() + assert len(adapter._session_webhooks) == 0 + assert len(adapter._seen_messages) == 0 + assert adapter._http_client is None + + +# --------------------------------------------------------------------------- +# Platform enum +# --------------------------------------------------------------------------- + + +class TestPlatformEnum: + + def test_dingtalk_in_platform_enum(self): + assert Platform.DINGTALK.value == "dingtalk" diff --git a/tests/tools/test_website_policy.py b/tests/tools/test_website_policy.py new file mode 100644 index 0000000000..13f7aedd6a --- /dev/null +++ b/tests/tools/test_website_policy.py @@ -0,0 +1,490 @@ +import json +from pathlib import Path + +import pytest +import yaml + +from tools.website_policy import WebsitePolicyError, check_website_access, load_website_blocklist + + +def test_load_website_blocklist_merges_config_and_shared_file(tmp_path): + shared = tmp_path / "community-blocklist.txt" + shared.write_text("# comment\nexample.org\nsub.bad.net\n", encoding="utf-8") + + config_path = tmp_path / "config.yaml" + config_path.write_text( + yaml.safe_dump( + { + "security": { + "website_blocklist": { + "enabled": True, + "domains": ["example.com", "https://www.evil.test/path"], + "shared_files": [str(shared)], + } + } + }, + sort_keys=False, + ), + encoding="utf-8", + ) + + policy = load_website_blocklist(config_path) + + assert policy["enabled"] is True + assert {rule["pattern"] for rule in policy["rules"]} == { + "example.com", + "evil.test", + "example.org", + "sub.bad.net", + } + + +def test_check_website_access_matches_parent_domain_subdomains(tmp_path): + config_path = tmp_path / "config.yaml" + config_path.write_text( + yaml.safe_dump( + { + "security": { + "website_blocklist": { + "enabled": True, + "domains": ["example.com"], + } + } + }, + sort_keys=False, + ), + encoding="utf-8", + ) + + blocked = check_website_access("https://docs.example.com/page", config_path=config_path) + + assert blocked is not None + assert blocked["host"] == "docs.example.com" + assert blocked["rule"] == "example.com" + + +def test_check_website_access_supports_wildcard_subdomains_only(tmp_path): + config_path = tmp_path / "config.yaml" + config_path.write_text( + yaml.safe_dump( + { + "security": { + "website_blocklist": { + "enabled": True, + "domains": ["*.tracking.example"], + } + } + }, + sort_keys=False, + ), + encoding="utf-8", + ) + + assert check_website_access("https://a.tracking.example", config_path=config_path) is not None + assert check_website_access("https://www.tracking.example", config_path=config_path) is not None + assert check_website_access("https://tracking.example", config_path=config_path) is None + + +def test_default_config_exposes_website_blocklist_shape(): + from hermes_cli.config import DEFAULT_CONFIG + + website_blocklist = DEFAULT_CONFIG["security"]["website_blocklist"] + assert website_blocklist["enabled"] is False + assert website_blocklist["domains"] == [] + assert website_blocklist["shared_files"] == [] + + +def test_load_website_blocklist_uses_enabled_default_when_section_missing(tmp_path): + config_path = tmp_path / "config.yaml" + config_path.write_text(yaml.safe_dump({"display": {"tool_progress": "all"}}, sort_keys=False), encoding="utf-8") + + policy = load_website_blocklist(config_path) + + assert policy == {"enabled": False, "rules": []} + + +def test_load_website_blocklist_raises_clean_error_for_invalid_domains_type(tmp_path): + config_path = tmp_path / "config.yaml" + config_path.write_text( + yaml.safe_dump( + { + "security": { + "website_blocklist": { + "enabled": True, + "domains": "example.com", + } + } + }, + sort_keys=False, + ), + encoding="utf-8", + ) + + with pytest.raises(WebsitePolicyError, match="security.website_blocklist.domains must be a list"): + load_website_blocklist(config_path) + + +def test_load_website_blocklist_raises_clean_error_for_invalid_shared_files_type(tmp_path): + config_path = tmp_path / "config.yaml" + config_path.write_text( + yaml.safe_dump( + { + "security": { + "website_blocklist": { + "enabled": True, + "shared_files": "community-blocklist.txt", + } + } + }, + sort_keys=False, + ), + encoding="utf-8", + ) + + with pytest.raises(WebsitePolicyError, match="security.website_blocklist.shared_files must be a list"): + load_website_blocklist(config_path) + + +def test_load_website_blocklist_raises_clean_error_for_invalid_top_level_config_type(tmp_path): + config_path = tmp_path / "config.yaml" + config_path.write_text(yaml.safe_dump(["not", "a", "mapping"], sort_keys=False), encoding="utf-8") + + with pytest.raises(WebsitePolicyError, match="config root must be a mapping"): + load_website_blocklist(config_path) + + +def test_load_website_blocklist_raises_clean_error_for_invalid_security_type(tmp_path): + config_path = tmp_path / "config.yaml" + config_path.write_text(yaml.safe_dump({"security": []}, sort_keys=False), encoding="utf-8") + + with pytest.raises(WebsitePolicyError, match="security must be a mapping"): + load_website_blocklist(config_path) + + +def test_load_website_blocklist_raises_clean_error_for_invalid_website_blocklist_type(tmp_path): + config_path = tmp_path / "config.yaml" + config_path.write_text( + yaml.safe_dump( + { + "security": { + "website_blocklist": "block everything", + } + }, + sort_keys=False, + ), + encoding="utf-8", + ) + + with pytest.raises(WebsitePolicyError, match="security.website_blocklist must be a mapping"): + load_website_blocklist(config_path) + + +def test_load_website_blocklist_raises_clean_error_for_invalid_enabled_type(tmp_path): + config_path = tmp_path / "config.yaml" + config_path.write_text( + yaml.safe_dump( + { + "security": { + "website_blocklist": { + "enabled": "false", + } + } + }, + sort_keys=False, + ), + encoding="utf-8", + ) + + with pytest.raises(WebsitePolicyError, match="security.website_blocklist.enabled must be a boolean"): + load_website_blocklist(config_path) + + +def test_load_website_blocklist_raises_clean_error_for_malformed_yaml(tmp_path): + config_path = tmp_path / "config.yaml" + config_path.write_text("security: [oops\n", encoding="utf-8") + + with pytest.raises(WebsitePolicyError, match="Invalid config YAML"): + load_website_blocklist(config_path) + + +def test_load_website_blocklist_wraps_shared_file_read_errors(tmp_path, monkeypatch): + shared = tmp_path / "community-blocklist.txt" + shared.write_text("example.org\n", encoding="utf-8") + + config_path = tmp_path / "config.yaml" + config_path.write_text( + yaml.safe_dump( + { + "security": { + "website_blocklist": { + "enabled": True, + "shared_files": [str(shared)], + } + } + }, + sort_keys=False, + ), + encoding="utf-8", + ) + + def failing_read_text(self, *args, **kwargs): + raise PermissionError("no permission") + + monkeypatch.setattr(Path, "read_text", failing_read_text) + + # Unreadable shared files are now warned and skipped (not raised), + # so the blocklist loads successfully but without those rules. + result = load_website_blocklist(config_path) + assert result["enabled"] is True + assert result["rules"] == [] # shared file rules skipped + + +def test_check_website_access_uses_dynamic_hermes_home(monkeypatch, tmp_path): + hermes_home = tmp_path / "hermes-home" + hermes_home.mkdir() + (hermes_home / "config.yaml").write_text( + yaml.safe_dump( + { + "security": { + "website_blocklist": { + "enabled": True, + "domains": ["dynamic.example"], + } + } + }, + sort_keys=False, + ), + encoding="utf-8", + ) + + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + + blocked = check_website_access("https://dynamic.example/path") + + assert blocked is not None + assert blocked["rule"] == "dynamic.example" + + +def test_check_website_access_blocks_scheme_less_urls(tmp_path): + config_path = tmp_path / "config.yaml" + config_path.write_text( + yaml.safe_dump( + { + "security": { + "website_blocklist": { + "enabled": True, + "domains": ["blocked.test"], + } + } + }, + sort_keys=False, + ), + encoding="utf-8", + ) + + blocked = check_website_access("www.blocked.test/path", config_path=config_path) + + assert blocked is not None + assert blocked["host"] == "www.blocked.test" + assert blocked["rule"] == "blocked.test" + + +def test_browser_navigate_returns_policy_block(monkeypatch): + from tools import browser_tool + + monkeypatch.setattr( + browser_tool, + "check_website_access", + lambda url: { + "host": "blocked.test", + "rule": "blocked.test", + "source": "config", + "message": "Blocked by website policy", + }, + ) + monkeypatch.setattr( + browser_tool, + "_run_browser_command", + lambda *args, **kwargs: pytest.fail("browser command should not run for blocked URL"), + ) + + result = json.loads(browser_tool.browser_navigate("https://blocked.test")) + + assert result["success"] is False + assert result["blocked_by_policy"]["rule"] == "blocked.test" + + +def test_browser_navigate_allows_when_shared_file_missing(monkeypatch, tmp_path): + """Missing shared blocklist files are warned and skipped, not fatal.""" + from tools import browser_tool + + config_path = tmp_path / "config.yaml" + config_path.write_text( + yaml.safe_dump( + { + "security": { + "website_blocklist": { + "enabled": True, + "shared_files": ["missing-blocklist.txt"], + } + } + }, + sort_keys=False, + ), + encoding="utf-8", + ) + + # check_website_access should return None (allow) — missing file is skipped + result = check_website_access("https://allowed.test", config_path=config_path) + assert result is None + + +@pytest.mark.asyncio +async def test_web_extract_short_circuits_blocked_url(monkeypatch): + from tools import web_tools + + monkeypatch.setattr( + web_tools, + "check_website_access", + lambda url: { + "host": "blocked.test", + "rule": "blocked.test", + "source": "config", + "message": "Blocked by website policy", + }, + ) + monkeypatch.setattr( + web_tools, + "_get_firecrawl_client", + lambda: pytest.fail("firecrawl should not run for blocked URL"), + ) + monkeypatch.setattr("tools.interrupt.is_interrupted", lambda: False) + + result = json.loads(await web_tools.web_extract_tool(["https://blocked.test"], use_llm_processing=False)) + + assert result["results"][0]["url"] == "https://blocked.test" + assert "Blocked by website policy" in result["results"][0]["error"] + + +def test_check_website_access_fails_open_on_malformed_config(tmp_path, monkeypatch): + """Malformed config with default path should fail open (return None), not crash.""" + config_path = tmp_path / "config.yaml" + config_path.write_text("security: [oops\n", encoding="utf-8") + + # With explicit config_path (test mode), errors propagate + with pytest.raises(WebsitePolicyError): + check_website_access("https://example.com", config_path=config_path) + + # Simulate default path by pointing HERMES_HOME to tmp_path + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + from tools import website_policy + website_policy.invalidate_cache() + + # With default path, errors are caught and fail open + result = check_website_access("https://example.com") + assert result is None # allowed, not crashed + + +@pytest.mark.asyncio +async def test_web_extract_blocks_redirected_final_url(monkeypatch): + from tools import web_tools + + def fake_check(url): + if url == "https://allowed.test": + return None + if url == "https://blocked.test/final": + return { + "host": "blocked.test", + "rule": "blocked.test", + "source": "config", + "message": "Blocked by website policy", + } + pytest.fail(f"unexpected URL checked: {url}") + + class FakeFirecrawlClient: + def scrape(self, url, formats): + return { + "markdown": "secret content", + "metadata": { + "title": "Redirected", + "sourceURL": "https://blocked.test/final", + }, + } + + monkeypatch.setattr(web_tools, "check_website_access", fake_check) + monkeypatch.setattr(web_tools, "_get_firecrawl_client", lambda: FakeFirecrawlClient()) + monkeypatch.setattr("tools.interrupt.is_interrupted", lambda: False) + + result = json.loads(await web_tools.web_extract_tool(["https://allowed.test"], use_llm_processing=False)) + + assert result["results"][0]["url"] == "https://blocked.test/final" + assert result["results"][0]["content"] == "" + assert result["results"][0]["blocked_by_policy"]["rule"] == "blocked.test" + + +@pytest.mark.asyncio +async def test_web_crawl_short_circuits_blocked_url(monkeypatch): + from tools import web_tools + + monkeypatch.setattr( + web_tools, + "check_website_access", + lambda url: { + "host": "blocked.test", + "rule": "blocked.test", + "source": "config", + "message": "Blocked by website policy", + }, + ) + monkeypatch.setattr( + web_tools, + "_get_firecrawl_client", + lambda: pytest.fail("firecrawl should not run for blocked crawl URL"), + ) + monkeypatch.setattr("tools.interrupt.is_interrupted", lambda: False) + + result = json.loads(await web_tools.web_crawl_tool("https://blocked.test", use_llm_processing=False)) + + assert result["results"][0]["url"] == "https://blocked.test" + assert result["results"][0]["blocked_by_policy"]["rule"] == "blocked.test" + + +@pytest.mark.asyncio +async def test_web_crawl_blocks_redirected_final_url(monkeypatch): + from tools import web_tools + + def fake_check(url): + if url == "https://allowed.test": + return None + if url == "https://blocked.test/final": + return { + "host": "blocked.test", + "rule": "blocked.test", + "source": "config", + "message": "Blocked by website policy", + } + pytest.fail(f"unexpected URL checked: {url}") + + class FakeCrawlClient: + def crawl(self, url, **kwargs): + return { + "data": [ + { + "markdown": "secret crawl content", + "metadata": { + "title": "Redirected crawl page", + "sourceURL": "https://blocked.test/final", + }, + } + ] + } + + monkeypatch.setattr(web_tools, "check_website_access", fake_check) + monkeypatch.setattr(web_tools, "_get_firecrawl_client", lambda: FakeCrawlClient()) + monkeypatch.setattr("tools.interrupt.is_interrupted", lambda: False) + + result = json.loads(await web_tools.web_crawl_tool("https://allowed.test", use_llm_processing=False)) + + assert result["results"][0]["content"] == "" + assert result["results"][0]["error"] == "Blocked by website policy" + assert result["results"][0]["blocked_by_policy"]["rule"] == "blocked.test" diff --git a/tools/browser_tool.py b/tools/browser_tool.py index d57eedee89..c396c6e506 100644 --- a/tools/browser_tool.py +++ b/tools/browser_tool.py @@ -65,6 +65,11 @@ import requests from typing import Dict, Any, Optional, List from pathlib import Path from agent.auxiliary_client import call_llm + +try: + from tools.website_policy import check_website_access +except Exception: + check_website_access = lambda url: None # noqa: E731 — fail-open if policy module unavailable from tools.browser_providers.base import CloudBrowserProvider from tools.browser_providers.browserbase import BrowserbaseProvider from tools.browser_providers.browser_use import BrowserUseProvider @@ -901,6 +906,15 @@ def browser_navigate(url: str, task_id: Optional[str] = None) -> str: Returns: JSON string with navigation result (includes stealth features info on first nav) """ + # Website policy check — block before navigating + blocked = check_website_access(url) + if blocked: + return json.dumps({ + "success": False, + "error": blocked["message"], + "blocked_by_policy": {"host": blocked["host"], "rule": blocked["rule"], "source": blocked["source"]}, + }) + effective_task_id = task_id or "default" # Get session info to check if this is a new session diff --git a/tools/fuzzy_match.py b/tools/fuzzy_match.py index f53451c63d..ddcdf4274e 100644 --- a/tools/fuzzy_match.py +++ b/tools/fuzzy_match.py @@ -254,10 +254,9 @@ def _strategy_trimmed_boundary(content: str, pattern: str) -> List[Tuple[int, in if '\n'.join(check_lines) == modified_pattern: # Found match - calculate original positions - start_pos = sum(len(line) + 1 for line in content_lines[:i]) - end_pos = sum(len(line) + 1 for line in content_lines[:i + pattern_line_count]) - 1 - if end_pos >= len(content): - end_pos = len(content) + start_pos, end_pos = _calculate_line_positions( + content_lines, i, i + pattern_line_count, len(content) + ) matches.append((start_pos, end_pos)) return matches @@ -309,9 +308,10 @@ def _strategy_block_anchor(content: str, pattern: str) -> List[Tuple[int, int]]: if similarity >= threshold: # Calculate positions using ORIGINAL lines to ensure correct character offsets in the file - start_pos = sum(len(line) + 1 for line in orig_content_lines[:i]) - end_pos = sum(len(line) + 1 for line in orig_content_lines[:i + pattern_line_count]) - 1 - matches.append((start_pos, min(end_pos, len(content)))) + start_pos, end_pos = _calculate_line_positions( + orig_content_lines, i, i + pattern_line_count, len(content) + ) + matches.append((start_pos, end_pos)) return matches @@ -343,10 +343,9 @@ def _strategy_context_aware(content: str, pattern: str) -> List[Tuple[int, int]] # Need at least 50% of lines to have high similarity if high_similarity_count >= len(pattern_lines) * 0.5: - start_pos = sum(len(line) + 1 for line in content_lines[:i]) - end_pos = sum(len(line) + 1 for line in content_lines[:i + pattern_line_count]) - 1 - if end_pos >= len(content): - end_pos = len(content) + start_pos, end_pos = _calculate_line_positions( + content_lines, i, i + pattern_line_count, len(content) + ) matches.append((start_pos, end_pos)) return matches @@ -356,6 +355,26 @@ def _strategy_context_aware(content: str, pattern: str) -> List[Tuple[int, int]] # Helper Functions # ============================================================================= +def _calculate_line_positions(content_lines: List[str], start_line: int, + end_line: int, content_length: int) -> Tuple[int, int]: + """Calculate start and end character positions from line indices. + + Args: + content_lines: List of lines (without newlines) + start_line: Starting line index (0-based) + end_line: Ending line index (exclusive, 0-based) + content_length: Total length of the original content string + + Returns: + Tuple of (start_pos, end_pos) in the original content + """ + start_pos = sum(len(line) + 1 for line in content_lines[:start_line]) + end_pos = sum(len(line) + 1 for line in content_lines[:end_line]) - 1 + if end_pos >= content_length: + end_pos = content_length + return start_pos, end_pos + + def _find_normalized_matches(content: str, content_lines: List[str], content_normalized_lines: List[str], pattern: str, pattern_normalized: str) -> List[Tuple[int, int]]: @@ -383,13 +402,9 @@ def _find_normalized_matches(content: str, content_lines: List[str], if block == pattern_normalized: # Found a match - calculate original positions - start_pos = sum(len(line) + 1 for line in content_lines[:i]) - end_pos = sum(len(line) + 1 for line in content_lines[:i + num_pattern_lines]) - 1 - - # Handle case where end is past content - if end_pos >= len(content): - end_pos = len(content) - + start_pos, end_pos = _calculate_line_positions( + content_lines, i, i + num_pattern_lines, len(content) + ) matches.append((start_pos, end_pos)) return matches diff --git a/tools/web_tools.py b/tools/web_tools.py index ede1adb038..92c0ae6011 100644 --- a/tools/web_tools.py +++ b/tools/web_tools.py @@ -49,6 +49,7 @@ from typing import List, Dict, Any, Optional from firecrawl import Firecrawl from agent.auxiliary_client import async_call_llm from tools.debug_helpers import DebugSession +from tools.website_policy import check_website_access logger = logging.getLogger(__name__) @@ -616,6 +617,17 @@ async def web_extract_tool( results.append({"url": url, "error": "Interrupted", "title": ""}) continue + # Website policy check — block before fetching + blocked = check_website_access(url) + if blocked: + logger.info("Blocked web_extract for %s by rule %s", blocked["host"], blocked["rule"]) + results.append({ + "url": url, "title": "", "content": "", + "error": blocked["message"], + "blocked_by_policy": {"host": blocked["host"], "rule": blocked["rule"], "source": blocked["source"]}, + }) + continue + try: logger.info("Scraping: %s", url) scrape_result = _get_firecrawl_client().scrape( @@ -669,11 +681,23 @@ async def web_extract_tool( # Get title from metadata title = metadata.get("title", "") + # Re-check final URL after redirect + final_url = metadata.get("sourceURL", url) + final_blocked = check_website_access(final_url) + if final_blocked: + logger.info("Blocked redirected web_extract for %s by rule %s", final_blocked["host"], final_blocked["rule"]) + results.append({ + "url": final_url, "title": title, "content": "", "raw_content": "", + "error": final_blocked["message"], + "blocked_by_policy": {"host": final_blocked["host"], "rule": final_blocked["rule"], "source": final_blocked["source"]}, + }) + continue + # Choose content based on requested format chosen_content = content_markdown if (format == "markdown" or (format is None and content_markdown)) else content_html or content_markdown or "" results.append({ - "url": metadata.get("sourceURL", url), + "url": final_url, "title": title, "content": chosen_content, "raw_content": chosen_content, @@ -778,6 +802,7 @@ async def web_extract_tool( "title": r.get("title", ""), "content": r.get("content", ""), "error": r.get("error"), + **({ "blocked_by_policy": r["blocked_by_policy"]} if "blocked_by_policy" in r else {}), } for r in response.get("results", []) ] @@ -870,6 +895,13 @@ async def web_crawl_tool( instructions_text = f" with instructions: '{instructions}'" if instructions else "" logger.info("Crawling %s%s", url, instructions_text) + # Website policy check — block before crawling + blocked = check_website_access(url) + if blocked: + logger.info("Blocked web_crawl for %s by rule %s", blocked["host"], blocked["rule"]) + return json.dumps({"results": [{"url": url, "title": "", "content": "", "error": blocked["message"], + "blocked_by_policy": {"host": blocked["host"], "rule": blocked["rule"], "source": blocked["source"]}}]}, ensure_ascii=False) + # Use Firecrawl's v2 crawl functionality # Docs: https://docs.firecrawl.dev/features/crawl # The crawl() method automatically waits for completion and returns all data @@ -975,6 +1007,17 @@ async def web_crawl_tool( page_url = metadata.get("sourceURL", metadata.get("url", "Unknown URL")) title = metadata.get("title", "") + # Re-check crawled page URL against policy + page_blocked = check_website_access(page_url) + if page_blocked: + logger.info("Blocked crawled page %s by rule %s", page_blocked["host"], page_blocked["rule"]) + pages.append({ + "url": page_url, "title": title, "content": "", "raw_content": "", + "error": page_blocked["message"], + "blocked_by_policy": {"host": page_blocked["host"], "rule": page_blocked["rule"], "source": page_blocked["source"]}, + }) + continue + # Choose content (prefer markdown) content = content_markdown or content_html or "" @@ -1070,9 +1113,11 @@ async def web_crawl_tool( # Trim output to minimal fields per entry: title, content, error trimmed_results = [ { + "url": r.get("url", ""), "title": r.get("title", ""), "content": r.get("content", ""), - "error": r.get("error") + "error": r.get("error"), + **({ "blocked_by_policy": r["blocked_by_policy"]} if "blocked_by_policy" in r else {}), } for r in response.get("results", []) ] diff --git a/tools/website_policy.py b/tools/website_policy.py new file mode 100644 index 0000000000..2a3d2470fb --- /dev/null +++ b/tools/website_policy.py @@ -0,0 +1,285 @@ +"""Website access policy helpers for URL-capable tools. + +This module loads a user-managed website blocklist from ~/.hermes/config.yaml +and optional shared list files. It is intentionally lightweight so web/browser +tools can enforce URL policy without pulling in the heavier CLI config stack. + +Policy is cached in memory with a short TTL so config changes take effect +quickly without re-reading the file on every URL check. +""" + +from __future__ import annotations + +import fnmatch +import logging +import os +import threading +import time +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple +from urllib.parse import urlparse + +logger = logging.getLogger(__name__) + +_DEFAULT_WEBSITE_BLOCKLIST = { + "enabled": False, + "domains": [], + "shared_files": [], +} + +# Cache: parsed policy + timestamp. Avoids re-reading config.yaml on every +# URL check (a web_crawl with 50 pages would otherwise mean 51 YAML parses). +_CACHE_TTL_SECONDS = 30.0 +_cache_lock = threading.Lock() +_cached_policy: Optional[Dict[str, Any]] = None +_cached_policy_path: Optional[str] = None +_cached_policy_time: float = 0.0 + + +def _get_hermes_home() -> Path: + return Path(os.getenv("HERMES_HOME", Path.home() / ".hermes")) + + +def _get_default_config_path() -> Path: + return _get_hermes_home() / "config.yaml" + + +class WebsitePolicyError(Exception): + """Raised when a website policy file is malformed.""" + + +def _normalize_host(host: str) -> str: + return (host or "").strip().lower().rstrip(".") + + +def _normalize_rule(rule: Any) -> Optional[str]: + if not isinstance(rule, str): + return None + value = rule.strip().lower() + if not value or value.startswith("#"): + return None + if "://" in value: + parsed = urlparse(value) + value = parsed.netloc or parsed.path + value = value.split("/", 1)[0].strip().rstrip(".") + if value.startswith("www."): + value = value[4:] + return value or None + + +def _iter_blocklist_file_rules(path: Path) -> List[str]: + """Load rules from a shared blocklist file. + + Missing or unreadable files log a warning and return an empty list + rather than raising — a bad file path should not disable all web tools. + """ + try: + raw = path.read_text(encoding="utf-8") + except FileNotFoundError: + logger.warning("Shared blocklist file not found (skipping): %s", path) + return [] + except (OSError, UnicodeDecodeError) as exc: + logger.warning("Failed to read shared blocklist file %s (skipping): %s", path, exc) + return [] + + rules: List[str] = [] + for line in raw.splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + normalized = _normalize_rule(stripped) + if normalized: + rules.append(normalized) + return rules + + +def _load_policy_config(config_path: Optional[Path] = None) -> Dict[str, Any]: + config_path = config_path or _get_default_config_path() + if not config_path.exists(): + return dict(_DEFAULT_WEBSITE_BLOCKLIST) + + try: + import yaml + except ImportError: + logger.debug("PyYAML not installed — website blocklist disabled") + return dict(_DEFAULT_WEBSITE_BLOCKLIST) + + try: + with open(config_path, encoding="utf-8") as f: + config = yaml.safe_load(f) or {} + except yaml.YAMLError as exc: + raise WebsitePolicyError(f"Invalid config YAML at {config_path}: {exc}") from exc + except OSError as exc: + raise WebsitePolicyError(f"Failed to read config file {config_path}: {exc}") from exc + if not isinstance(config, dict): + raise WebsitePolicyError("config root must be a mapping") + + security = config.get("security", {}) + if security is None: + security = {} + if not isinstance(security, dict): + raise WebsitePolicyError("security must be a mapping") + + website_blocklist = security.get("website_blocklist", {}) + if website_blocklist is None: + website_blocklist = {} + if not isinstance(website_blocklist, dict): + raise WebsitePolicyError("security.website_blocklist must be a mapping") + + policy = dict(_DEFAULT_WEBSITE_BLOCKLIST) + policy.update(website_blocklist) + return policy + + +def load_website_blocklist(config_path: Optional[Path] = None) -> Dict[str, Any]: + """Load and return the parsed website blocklist policy. + + Results are cached for ``_CACHE_TTL_SECONDS`` to avoid re-reading + config.yaml on every URL check. Pass an explicit ``config_path`` + to bypass the cache (used by tests). + """ + global _cached_policy, _cached_policy_path, _cached_policy_time + + resolved_path = str(config_path) if config_path else "__default__" + now = time.monotonic() + + # Return cached policy if still fresh and same path + if config_path is None: + with _cache_lock: + if ( + _cached_policy is not None + and _cached_policy_path == resolved_path + and (now - _cached_policy_time) < _CACHE_TTL_SECONDS + ): + return _cached_policy + + config_path = config_path or _get_default_config_path() + policy = _load_policy_config(config_path) + + raw_domains = policy.get("domains", []) or [] + if not isinstance(raw_domains, list): + raise WebsitePolicyError("security.website_blocklist.domains must be a list") + + raw_shared_files = policy.get("shared_files", []) or [] + if not isinstance(raw_shared_files, list): + raise WebsitePolicyError("security.website_blocklist.shared_files must be a list") + + enabled = policy.get("enabled", True) + if not isinstance(enabled, bool): + raise WebsitePolicyError("security.website_blocklist.enabled must be a boolean") + + rules: List[Dict[str, str]] = [] + seen: set[Tuple[str, str]] = set() + + for raw_rule in raw_domains: + normalized = _normalize_rule(raw_rule) + if normalized and ("config", normalized) not in seen: + rules.append({"pattern": normalized, "source": "config"}) + seen.add(("config", normalized)) + + for shared_file in raw_shared_files: + if not isinstance(shared_file, str) or not shared_file.strip(): + continue + path = Path(shared_file).expanduser() + if not path.is_absolute(): + path = (_get_hermes_home() / path).resolve() + for normalized in _iter_blocklist_file_rules(path): + key = (str(path), normalized) + if key in seen: + continue + rules.append({"pattern": normalized, "source": str(path)}) + seen.add(key) + + result = {"enabled": enabled, "rules": rules} + + # Cache the result (only for the default path — explicit paths are tests) + if config_path == _get_default_config_path(): + with _cache_lock: + _cached_policy = result + _cached_policy_path = "__default__" + _cached_policy_time = now + + return result + + +def invalidate_cache() -> None: + """Force the next ``check_website_access`` call to re-read config.""" + global _cached_policy + with _cache_lock: + _cached_policy = None + + +def _match_host_against_rule(host: str, pattern: str) -> bool: + if not host or not pattern: + return False + if pattern.startswith("*."): + return fnmatch.fnmatch(host, pattern) + return host == pattern or host.endswith(f".{pattern}") + + +def _extract_host_from_urlish(url: str) -> str: + parsed = urlparse(url) + host = _normalize_host(parsed.hostname or parsed.netloc) + if host: + return host + + if "://" not in url: + schemeless = urlparse(f"//{url}") + host = _normalize_host(schemeless.hostname or schemeless.netloc) + if host: + return host + + return "" + + +def check_website_access(url: str, config_path: Optional[Path] = None) -> Optional[Dict[str, str]]: + """Check whether a URL is allowed by the website blocklist policy. + + Returns ``None`` if access is allowed, or a dict with block metadata + (``host``, ``rule``, ``source``, ``message``) if blocked. + + Never raises on policy errors — logs a warning and returns ``None`` + (fail-open) so a config typo doesn't break all web tools. Pass + ``config_path`` explicitly (tests) to get strict error propagation. + """ + # Fast path: if no explicit config_path and the cached policy is disabled + # or empty, skip all work (no YAML read, no host extraction). + if config_path is None: + with _cache_lock: + if _cached_policy is not None and not _cached_policy.get("enabled"): + return None + + host = _extract_host_from_urlish(url) + if not host: + return None + + try: + policy = load_website_blocklist(config_path) + except WebsitePolicyError as exc: + if config_path is not None: + raise # Tests pass explicit paths — let errors propagate + logger.warning("Website policy config error (failing open): %s", exc) + return None + except Exception as exc: + logger.warning("Unexpected error loading website policy (failing open): %s", exc) + return None + + if not policy.get("enabled"): + return None + + for rule in policy.get("rules", []): + pattern = rule.get("pattern", "") + if _match_host_against_rule(host, pattern): + logger.info("Blocked URL %s — matched rule '%s' from %s", + url, pattern, rule.get("source", "config")) + return { + "url": url, + "host": host, + "rule": pattern, + "source": rule.get("source", "config"), + "message": ( + f"Blocked by website policy: '{host}' matched rule '{pattern}'" + f" from {rule.get('source', 'config')}" + ), + } + return None diff --git a/website/docs/reference/environment-variables.md b/website/docs/reference/environment-variables.md index b1028588f2..3d8f5d8139 100644 --- a/website/docs/reference/environment-variables.md +++ b/website/docs/reference/environment-variables.md @@ -32,6 +32,7 @@ All variables go in `~/.hermes/.env`. You can also set them with `hermes config | `KILOCODE_BASE_URL` | Override Kilo Code base URL (default: `https://api.kilo.ai/api/gateway`) | | `ANTHROPIC_API_KEY` | Anthropic Console API key ([console.anthropic.com](https://console.anthropic.com/)) | | `ANTHROPIC_TOKEN` | Manual or legacy Anthropic OAuth/setup-token override | +| `DASHSCOPE_API_KEY` | Alibaba Cloud DashScope API key for Qwen models via Anthropic-compatible API ([modelstudio.console.alibabacloud.com](https://modelstudio.console.alibabacloud.com/)) | | `CLAUDE_CODE_OAUTH_TOKEN` | Explicit Claude Code token override if you export one manually | | `HERMES_MODEL` | Preferred model name (checked before `LLM_MODEL`, used by gateway) | | `LLM_MODEL` | Default model name (fallback when not set in config.yaml) | @@ -46,7 +47,7 @@ For native Anthropic auth, Hermes prefers Claude Code's own credential files whe | Variable | Description | |----------|-------------| -| `HERMES_INFERENCE_PROVIDER` | Override provider selection: `auto`, `openrouter`, `nous`, `openai-codex`, `anthropic`, `zai`, `kimi-coding`, `minimax`, `minimax-cn`, `kilocode` (default: `auto`) | +| `HERMES_INFERENCE_PROVIDER` | Override provider selection: `auto`, `openrouter`, `nous`, `openai-codex`, `anthropic`, `zai`, `kimi-coding`, `minimax`, `minimax-cn`, `kilocode`, `alibaba` (default: `auto`) | | `HERMES_PORTAL_BASE_URL` | Override Nous Portal URL (for development/testing) | | `NOUS_INFERENCE_BASE_URL` | Override Nous inference API URL | | `HERMES_NOUS_MIN_KEY_TTL_SECONDS` | Min agent key TTL before re-mint (default: 1800 = 30min) | @@ -170,6 +171,8 @@ For native Anthropic auth, Hermes prefers Claude Code's own credential files whe | `EMAIL_HOME_ADDRESS_NAME` | Display name for the email home target | | `EMAIL_POLL_INTERVAL` | Email polling interval in seconds | | `EMAIL_ALLOW_ALL_USERS` | Allow all inbound email senders | +| `DINGTALK_CLIENT_ID` | DingTalk bot AppKey from developer portal ([open.dingtalk.com](https://open.dingtalk.com)) | +| `DINGTALK_CLIENT_SECRET` | DingTalk bot AppSecret from developer portal | | `HASS_TOKEN` | Home Assistant Long-Lived Access Token (enables HA platform + tools) | | `HASS_URL` | Home Assistant URL (default: `http://homeassistant.local:8123`) | | `MESSAGING_CWD` | Working directory for terminal commands in messaging mode (default: `~`) |