mirror of
https://github.com/TencentCloud/Octop.git
synced 2026-09-29 03:43:01 +08:00
* fix: schedule proactive care for agents that never saved config Proactive care defaults to on, but the scheduler only listed persisted enabled rows, so most experts never entered the push loop. * fix: attach gateway connector tools without rebuilding the agent prepare_chat_mcp rebuilt the agent whenever a builtin MCP server was missing from the live runtime. For gateway-mode connectors that rebuild is pure loss: they carry no HTTP transport, their tools are built in-process from stored credentials, and _post_start_agent runs the very same injection at the end anyway. Meanwhile the rebuild drops the harness instance and with it the checkpointer pool an in-flight turn is still writing to. Split the missing names on connector mode: gateway ones refresh their credentials and inject into the running agent, the rest keep the existing reload path. Co-authored-by: Cursor <cursoragent@cursor.com> * fix: await cancellation on scheduler shutdown, and keep its sleeps out of tests ProactiveCareScheduler.shutdown() cancelled each task and returned without awaiting it, so a loop parked in a multi-hour sleep could outlive the shutdown that was supposed to end it. It now gathers the cancelled tasks before returning. Proactive care defaults to on, so creating an agent starts one of those sleeps. pytest-asyncio waits for leftover tasks before fixture teardown, which hangs the suite. octop_client shuts the scheduler down and calls the new suspend() so nothing reschedules; an autouse fixture covers tests that boot OctopServer another way. The scheduler's own unit tests opt out. suspend() exists for the tests. The alternative — every test remembering to tear the scheduler down — is the arrangement that produced the hang. * refactor: drive websockets on one event loop and drop the cross-loop workaround starlette's sync TestClient runs the app on its own anyio portal loop. That left the process with two loops over one OctopServer, so primitives bound to a loop and shared by both — AgentManager._lock — deadlocked or raised "bound to a different event loop". The tests worked around it by running the ws session in a worker thread. tests.support.http now speaks ASGI directly, so the handler, the gateway workers and the test all share one loop, the way uvicorn runs it in production. That removes the reason the production handlers marshalled every outbound frame across loops with run_coroutine_threadsafe + wrap_future. The comment on that code named the cause outright — "this handler may run on a different loop (e.g. starlette's TestClient portal)" — so it was test-shaped machinery sitting on the path of every frame the dashboard receives. Both handlers now await the send directly. * update harness-agent & harness-memory ---------
165 lines
6.0 KiB
Python
165 lines
6.0 KiB
Python
"""WebSocket / streaming helpers for integration tests."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import contextlib
|
|
import json
|
|
from collections.abc import AsyncIterator
|
|
from contextlib import asynccontextmanager
|
|
from typing import Any
|
|
from urllib.parse import unquote, urlsplit
|
|
|
|
import httpx
|
|
from starlette.websockets import WebSocketDisconnect
|
|
|
|
# A single-loop session should never stall; the cap keeps a regression from
|
|
# wedging the whole suite instead of failing this one test.
|
|
_WS_RECEIVE_TIMEOUT_S = 20.0
|
|
|
|
|
|
def ws_token(auth: dict[str, str]) -> str:
|
|
return auth["Authorization"].split(" ", 1)[1]
|
|
|
|
|
|
class ASGIWebSocketSession:
|
|
"""Drive an ASGI websocket endpoint on the caller's event loop.
|
|
|
|
starlette's sync ``TestClient`` runs the app on its own anyio portal loop.
|
|
That gives the process two loops over one ``OctopServer``, and loop-bound
|
|
primitives shared by both (``AgentManager._lock``) then deadlock or raise
|
|
"bound to a different event loop". Speaking ASGI directly keeps the
|
|
handler, the gateway workers and the test on the same loop, exactly like
|
|
uvicorn in production.
|
|
"""
|
|
|
|
def __init__(self, app: Any, path: str) -> None:
|
|
self._app = app
|
|
self._path = path
|
|
self._from_app: asyncio.Queue[dict[str, Any]] = asyncio.Queue()
|
|
self._to_app: asyncio.Queue[dict[str, Any]] = asyncio.Queue()
|
|
self._task: asyncio.Task[None] | None = None
|
|
|
|
def _scope(self) -> dict[str, Any]:
|
|
url = urlsplit(self._path)
|
|
return {
|
|
"type": "websocket",
|
|
"asgi": {"version": "3.0", "spec_version": "2.3"},
|
|
"http_version": "1.1",
|
|
"scheme": "ws",
|
|
"path": unquote(url.path),
|
|
"raw_path": url.path.encode("ascii"),
|
|
"query_string": url.query.encode("ascii"),
|
|
"root_path": "",
|
|
"headers": [(b"host", b"testserver")],
|
|
"client": ("testclient", 50000),
|
|
"server": ("testserver", 80),
|
|
"subprotocols": [],
|
|
"state": {},
|
|
"app": self._app,
|
|
}
|
|
|
|
async def _next_event(self, timeout: float) -> dict[str, Any]:
|
|
assert self._task is not None
|
|
getter: asyncio.Future[dict[str, Any]] = asyncio.ensure_future(self._from_app.get())
|
|
done, _pending = await asyncio.wait(
|
|
{getter, self._task},
|
|
timeout=timeout,
|
|
return_when=asyncio.FIRST_COMPLETED,
|
|
)
|
|
if getter in done:
|
|
return getter.result()
|
|
getter.cancel()
|
|
with contextlib.suppress(asyncio.CancelledError):
|
|
await getter
|
|
if self._task in done:
|
|
# Surface a handler crash instead of a bare timeout.
|
|
self._task.result()
|
|
raise AssertionError("websocket handler returned without sending an event")
|
|
raise TimeoutError(f"no websocket event after {timeout:.0f}s")
|
|
|
|
async def connect(self, timeout: float = _WS_RECEIVE_TIMEOUT_S) -> None:
|
|
self._task = asyncio.create_task(
|
|
self._app(self._scope(), self._to_app.get, self._from_app.put),
|
|
name=f"asgi-ws{self._path}",
|
|
)
|
|
await self._to_app.put({"type": "websocket.connect"})
|
|
event = await self._next_event(timeout)
|
|
if event["type"] == "websocket.close":
|
|
raise WebSocketDisconnect(event.get("code", 1000), event.get("reason"))
|
|
if event["type"] != "websocket.accept":
|
|
raise AssertionError(f"unexpected handshake event: {event['type']}")
|
|
|
|
async def send_json(self, payload: Any) -> None:
|
|
await self._to_app.put(
|
|
{"type": "websocket.receive", "text": json.dumps(payload, ensure_ascii=False)},
|
|
)
|
|
|
|
async def receive_json(self, *, timeout: float = _WS_RECEIVE_TIMEOUT_S) -> dict[str, Any]:
|
|
event = await self._next_event(timeout)
|
|
if event["type"] == "websocket.close":
|
|
raise WebSocketDisconnect(event.get("code", 1000), event.get("reason"))
|
|
raw = event.get("text")
|
|
if raw is None:
|
|
raw = bytes(event.get("bytes") or b"").decode("utf-8")
|
|
parsed: dict[str, Any] = json.loads(raw)
|
|
return parsed
|
|
|
|
async def drain_turn(self, *, timeout: float = _WS_RECEIVE_TIMEOUT_S) -> list[dict[str, Any]]:
|
|
"""Collect frames until the turn reports ``done`` or ``error``."""
|
|
chunks: list[dict[str, Any]] = []
|
|
while True:
|
|
chunk = await self.receive_json(timeout=timeout)
|
|
chunks.append(chunk)
|
|
if chunk.get("type") in ("done", "error"):
|
|
return chunks
|
|
|
|
async def close(self) -> None:
|
|
if self._task is None:
|
|
return
|
|
task = self._task
|
|
self._task = None
|
|
await self._to_app.put({"type": "websocket.disconnect", "code": 1000})
|
|
try:
|
|
await asyncio.wait_for(task, timeout=_WS_RECEIVE_TIMEOUT_S)
|
|
except (asyncio.CancelledError, TimeoutError):
|
|
task.cancel()
|
|
with contextlib.suppress(asyncio.CancelledError):
|
|
await task
|
|
|
|
|
|
@asynccontextmanager
|
|
async def ws_connect(app: Any, path: str) -> AsyncIterator[ASGIWebSocketSession]:
|
|
"""Open an in-process websocket session against *app* on the current loop."""
|
|
session = ASGIWebSocketSession(app, path)
|
|
try:
|
|
await session.connect()
|
|
yield session
|
|
finally:
|
|
await session.close()
|
|
|
|
|
|
def chat_ws_path(agent_id: str, auth: dict[str, str]) -> str:
|
|
return f"/api/agents/{agent_id}/chat/ws?token={ws_token(auth)}"
|
|
|
|
|
|
async def ws_chat_turn(
|
|
client: httpx.AsyncClient,
|
|
agent_id: str,
|
|
auth: dict[str, str],
|
|
*,
|
|
mcp_servers: list[str] | None = None,
|
|
text: str = "hi",
|
|
) -> list[dict[str, Any]]:
|
|
body: dict[str, Any] = {
|
|
"type": "user_turn",
|
|
"text": text,
|
|
"messages": [{"role": "user", "content": text}],
|
|
}
|
|
if mcp_servers is not None:
|
|
body["mcp_servers"] = mcp_servers
|
|
app = client._octop_app # type: ignore[attr-defined]
|
|
async with ws_connect(app, chat_ws_path(agent_id, auth)) as ws:
|
|
await ws.send_json(body)
|
|
return await ws.drain_turn()
|