zzos/hermes-agent
1
0
Fork 0
mirror of https://github.com/NousResearch/hermes-agent.git synced 2026-09-28 06:45:17 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(aux): scope the response_format rejection memo to the model and to capability errors

Browse Source 
The process-level memo of routes that rejected a response_format type was
keyed by (endpoint host, type) only. On an aggregator host (openrouter.ai,
the Nous Portal, api.openai.com) one chat-only model's json_schema 400 then
stripped the field from every other model on that host for the rest of the
process, silently degrading schema enforcement for models that support it.
The memo was also fed by any error _is_structured_output_rejection matched,
including the by-shape rule (response_format + json_schema in the body),
which also matches schema-VALIDATION 400s from json_schema-capable providers
("Invalid schema for response_format 'json_schema': additionalProperties
must be false") - those say nothing about the next schema.

- key the memo by (endpoint, model, type); _build_call_kwargs passes the
  model to the lookup and every remember_* caller (both fallback candidate
  paths and the primary ladder rung) now hands over the rejecting error
- memoise only capability rejections (unavailable / not supported / unknown
  or unsupported parameter / extra inputs not permitted); the by-shape match
  still drives the one-shot retry but never feeds the memo
- a custom route whose base_url host is api.deepseek.com consults the
  DeepSeek profile (via the existing base_url->provider inference), so the
  profile gate also fires when the provider name was overridden to custom
- test module docstring: only title_generator and plugin_llm attach
  response_format on aux calls
This commit is contained in:
teknium1
2026-09-17 09:11:38 -07:00
committed by Teknium
parent df1074b4e5
commit 5e17f210c1
3 changed files with 96 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/auxiliary_client.py
+4 -4
View File
@@ -4019,7 +4019,7 @@ def _call_fallback_candidate_sync(
retry_kwargs = _fallback_structured_output_retry_kwargs(fb_err, fb_kwargs, task, fb_label)
if retry_kwargs is not None:
resp = _send(fb_client, retry_kwargs, destination)
remember_structured_output_rejection(destination.provider, destination.base_url, fb_kwargs)
remember_structured_output_rejection(destination.provider, destination.base_url, fb_kwargs, fb_err)
return resp
if not _is_auth_error(fb_err):
capacity = fallback_candidate_unavailable_reason(fb_err)
@@ -4074,7 +4074,7 @@ async def _call_fallback_candidate_async(
retry_kwargs = _fallback_structured_output_retry_kwargs(fb_err, fb_kwargs, task, fb_label)
if retry_kwargs is not None:
resp = await _send(fb_client, retry_kwargs, destination)
remember_structured_output_rejection(destination.provider, destination.base_url, fb_kwargs)
remember_structured_output_rejection(destination.provider, destination.base_url, fb_kwargs, fb_err)
return resp
if not _is_auth_error(fb_err):
capacity = fallback_candidate_unavailable_reason(fb_err)
@@ -6501,7 +6501,7 @@ def _build_call_kwargs(
merged_extra = _merge_aux_extra_body(extra_body, projection, reasoning_config, provider_norm)
if "response_format" in merged_extra:
from agent.auxiliary_structured_output import without_unsupported_response_format
merged_extra = without_unsupported_response_format(merged_extra, provider_norm, effective_base, task)
merged_extra = without_unsupported_response_format(merged_extra, provider_norm, effective_base, model, task)
if merged_extra:
kwargs["extra_body"] = merged_extra
# Anthropic Messages adapters take reasoning via a private kwarg that plain OpenAI SDK clients
@@ -7302,7 +7302,7 @@ def _ladder_parameter_rungs(
_LadderStep("call", (client, retry_kwargs)), _param_rung_accepts)
if first_err is None:
if remember is not None:
remember(route.resolved_provider, route.base_info, kwargs)
remember(route.resolved_provider, route.base_info, kwargs, rejection)
return resp, None, retry_kwargs
kwargs = retry_kwargs
return None, first_err, kwargs
agent/auxiliary_structured_output.py
+49 -17
View File
@@ -4,14 +4,21 @@ Two sources decide whether an aux request may carry a ``response_format`` type u
* the provider profile's ``unsupported_response_formats`` (DeepSeek's native API implements only
``json_object`` — https://api-docs.deepseek.com/guides/json_mode — and answers ``json_schema`` with
HTTP 400 "This response_format type is unavailable now"), and
* a process-level memo of routes that already rejected a type once; the recovery ladder records the
route when its retry without the field succeeded.
HTTP 400 "This response_format type is unavailable now"), also consulted when a ``custom`` route's
base_url points at a profiled provider's host, and
* a process-level memo of (endpoint, model, type) triples that already rejected the type once; the
recovery ladder records the triple when its retry without the field succeeded.
Either way the field is dropped before the first request instead of burning a guaranteed-fail
round-trip per call (#83390, #105191, #113064). Dropping — not downgrading to ``json_object`` — is the
same end state the rejection retry already produces: ``json_object`` needs the prompt to mention JSON
and some relays return empty content under it, so callers already tolerate prompt compliance.
The memo carries the model because capability is per model on aggregators (openrouter.ai, the Nous
Portal, api.openai.com host dozens of models with different structured-output support), and it is
fed only by rejections that name the *capability* — not by schema-validation 400s from providers that
do implement ``json_schema`` ("Invalid schema for response_format 'json_schema': additionalProperties
must be false"), which the ladder still retries once but which say nothing about the next schema.
"""
from __future__ import annotations
@@ -21,8 +28,15 @@ from urllib.parse import urlparse
logger = logging.getLogger(__name__)
# (route key, response_format type) pairs a provider rejected in this process.
_REJECTED_ROUTES: set[tuple[str, str]] = set()
# (route key, model, response_format type) triples a provider rejected in this process.
_REJECTED_ROUTES: set[tuple[str, str, str]] = set()
# Rejections that describe the route/model's capability rather than this request's schema.
_CAPABILITY_REJECTION_MARKERS = (
"unavailable", "does not support", "doesn't support", "not supported", "unsupported parameter",
"unsupported_parameter", "unknown parameter", "unrecognized request argument", "unrecognized parameter",
"extra inputs are not permitted",
)
def _route_key(provider: Optional[str], base_url: Optional[str]) -> str:
@@ -39,41 +53,59 @@ def _response_format_type(request_kwargs: Dict[str, Any]) -> Optional[str]:
return response_format.get("type") if isinstance(response_format, dict) else None
def _profile_unsupported_formats(provider: Optional[str]) -> tuple:
def _profile_unsupported_formats(provider: Optional[str], base_url: Optional[str]) -> tuple:
"""The provider profile's declared unsupported types; a ``custom`` route whose base_url is a profiled
provider's own host (``api.deepseek.com``) gets that provider's profile."""
try:
from providers import get_provider_profile
profile = get_provider_profile(str(provider or "").strip().lower())
name = str(provider or "").strip().lower()
if name == "custom" and base_url:
from agent.model_metadata import _infer_provider_from_url
name = _infer_provider_from_url(base_url) or name
profile = get_provider_profile(name)
except Exception:
return ()
return tuple(getattr(profile, "unsupported_response_formats", ()) or ()) if profile is not None else ()
def is_capability_rejection(error: Optional[BaseException]) -> bool:
"""Whether a structured-output rejection speaks to the route/model's capability (memoisable) rather
than to this request's schema (retry once, remember nothing)."""
err_lower = str(error or "").lower()
if "invalid schema" in err_lower:
return False
return any(marker in err_lower for marker in _CAPABILITY_REJECTION_MARKERS)
def remember_structured_output_rejection(
provider: Optional[str], base_url: Optional[str], rejected_kwargs: Dict[str, Any],
provider: Optional[str], base_url: Optional[str], rejected_kwargs: Dict[str, Any], error: BaseException,
) -> None:
"""Record that this route rejected the ``response_format`` type carried by *rejected_kwargs*."""
"""Record that this route's ``rejected_kwargs["model"]`` rejected the ``response_format`` type carried
by *rejected_kwargs* — only when *error* names the capability, never for a schema-validation 400."""
format_type = _response_format_type(rejected_kwargs)
if format_type:
_REJECTED_ROUTES.add((_route_key(provider, base_url), format_type))
if format_type and is_capability_rejection(error):
_REJECTED_ROUTES.add((_route_key(provider, base_url), str(rejected_kwargs.get("model") or ""), format_type))
def without_unsupported_response_format(
extra_body: Dict[str, Any], provider: Optional[str], base_url: Optional[str], task: Optional[str] = None,
extra_body: Dict[str, Any], provider: Optional[str], base_url: Optional[str], model: Optional[str],
task: Optional[str] = None,
) -> Dict[str, Any]:
"""*extra_body* minus a ``response_format`` whose type this route is known to reject; unchanged otherwise."""
"""*extra_body* minus a ``response_format`` whose type this route+model is known to reject; unchanged
otherwise."""
response_format = extra_body.get("response_format")
format_type = response_format.get("type") if isinstance(response_format, dict) else None
if not format_type:
return extra_body
known_unsupported = (
format_type in _profile_unsupported_formats(provider)
or (_route_key(provider, base_url), format_type) in _REJECTED_ROUTES
format_type in _profile_unsupported_formats(provider, base_url)
or (_route_key(provider, base_url), str(model or ""), format_type) in _REJECTED_ROUTES
)
if not known_unsupported:
return extra_body
logger.info(
"Auxiliary %s: %s does not accept response_format %s; sending without it "
"Auxiliary %s: %s (%s) does not accept response_format %s; sending without it "
"(schema enforcement degrades to prompt compliance)",
task or "call", _route_key(provider, base_url) or "provider", format_type,
task or "call", _route_key(provider, base_url) or "provider", model or "model", format_type,
)
return {k: v for k, v in extra_body.items() if k != "response_format"}
tests/agent/test_auxiliary_structured_output.py
+43 -5
View File
@@ -1,9 +1,10 @@
"""Structured-output (``response_format``) handling on auxiliary routes that reject it.
Covers NousResearch/hermes-agent#83390 / #105191 / #113064: a fallback candidate that rejects
``json_schema`` gets the same retry-without-the-field rung as the primary path instead of aborting
the task, and a route known to reject a ``response_format`` type (provider profile or a rejection
already seen in this process) never pays the guaranteed-fail first request.
Covers NousResearch/hermes-agent#83390 / #105191 / #113064 for the aux callers that attach
``response_format`` (``agent/title_generator.py`` and ``agent/plugin_llm.py``): a fallback candidate
that rejects ``json_schema`` gets the same retry-without-the-field rung as the primary path instead of
aborting the task, and a route+model known to reject a ``response_format`` type (provider profile or a
capability rejection already seen in this process) never pays the guaranteed-fail first request.
"""
import asyncio
from types import SimpleNamespace
@@ -16,6 +17,7 @@ from agent.auxiliary_client import (
_build_call_kwargs,
_call_fallback_candidate_async,
_call_fallback_candidate_sync,
_is_structured_output_rejection,
)
_JSON_SCHEMA = {"type": "json_schema", "json_schema": {"name": "t", "strict": True, "schema": {"type": "object"}}}
@@ -92,10 +94,46 @@ def test_known_unsupported_route_skips_response_format_before_first_request():
# A remembered rejection is scoped to the endpoint (host:port), not to every local server.
structured_output.remember_structured_output_rejection(
"custom", "http://127.0.0.1:1234/v1", {"extra_body": {"response_format": dict(_JSON_SCHEMA)}})
"custom", "http://127.0.0.1:1234/v1",
{"model": "m", "extra_body": {"response_format": dict(_JSON_SCHEMA)}}, _Rejects400(_DEEPSEEK_400))
same = _build_call_kwargs("custom", "m", messages, extra_body={"response_format": dict(_JSON_SCHEMA)},
base_url="http://127.0.0.1:1234/v1")
other = _build_call_kwargs("custom", "m", messages, extra_body={"response_format": dict(_JSON_SCHEMA)},
base_url="http://127.0.0.1:11434/v1")
assert "response_format" not in same.get("extra_body", {})
assert other["extra_body"]["response_format"] == _JSON_SCHEMA
# A custom route pointed at DeepSeek's own host gets DeepSeek's profile.
custom_deepseek = _build_call_kwargs(
"custom", "deepseek-chat", messages, extra_body={"response_format": dict(_JSON_SCHEMA)},
base_url="https://api.deepseek.com/v1")
assert "response_format" not in custom_deepseek.get("extra_body", {})
def test_rejection_memo_is_per_model_and_ignores_schema_validation_errors():
"""One model's ``json_schema`` rejection on an aggregator host must not strip the field for every other
model on that host, and a schema-VALIDATION 400 from a json_schema-capable provider is retried but
never memoised (it says nothing about the next schema)."""
messages = [{"role": "user", "content": "hi"}]
openrouter = "https://openrouter.ai/api/v1"
structured_output.remember_structured_output_rejection(
"openrouter", openrouter,
{"model": "some/chat-only-model", "extra_body": {"response_format": dict(_JSON_SCHEMA)}},
_Rejects400("Error code: 400 - {'error': {'message': 'response_format is not supported by this model'}}"))
rejected = _build_call_kwargs("openrouter", "some/chat-only-model", messages,
extra_body={"response_format": dict(_JSON_SCHEMA)}, base_url=openrouter)
sibling = _build_call_kwargs("openrouter", "openai/gpt-5-mini", messages,
extra_body={"response_format": dict(_JSON_SCHEMA)}, base_url=openrouter)
assert "response_format" not in rejected.get("extra_body", {})
assert sibling["extra_body"]["response_format"] == _JSON_SCHEMA
validation_400 = _Rejects400(
"Error code: 400 - Invalid schema for response_format 'json_schema': In context=(), "
"'additionalProperties' is required to be supplied and to be false.")
assert _is_structured_output_rejection(validation_400) # still drives the one-shot retry ...
structured_output.remember_structured_output_rejection(
"openai", "https://api.openai.com/v1",
{"model": "gpt-5-mini", "extra_body": {"response_format": dict(_JSON_SCHEMA)}}, validation_400)
after = _build_call_kwargs("openai", "gpt-5-mini", messages,
extra_body={"response_format": dict(_JSON_SCHEMA)}, base_url="https://api.openai.com/v1")
assert after["extra_body"]["response_format"] == _JSON_SCHEMA # ... but never feeds the memo