mirror of
https://github.com/OpenByteInc/QuantDinger.git
synced 2026-09-28 15:22:56 +08:00
873 lines
32 KiB
Python
873 lines
32 KiB
Python
from __future__ import annotations
|
|
|
|
import inspect
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import pandas as pd
|
|
import pytest
|
|
from flask import Flask, g
|
|
|
|
from app.services.strategy_evolution import (
|
|
EvolutionConfig,
|
|
SearchParameter,
|
|
StrategyEvolutionEngine,
|
|
StrategyEvolutionFailure,
|
|
StrategyEvolutionService,
|
|
)
|
|
from app.services.strategy_evolution.search import ParameterSampler
|
|
from app.services.strategy_evolution.statistics import (
|
|
deflated_sharpe,
|
|
equity_returns,
|
|
monte_carlo_bootstrap,
|
|
probability_of_backtest_overfitting,
|
|
)
|
|
from app.services.strategy_evolution.walk_forward import build_walk_forward_plan
|
|
from app.services.strategy_evolution import service as evolution_service_module
|
|
from app.services.strategy_evolution.evaluator import PreparedEvolutionEvaluator
|
|
from app.services.strategy_evolution.constraints import discover_parameter_constraints
|
|
from app.services.strategy_evolution.parameter_space import adapt_source_for_parameters, build_parameter_space
|
|
from app.services.strategy_v2 import StrategyV2BacktestService
|
|
from app.routes import strategy_evolution as evolution_routes
|
|
from app.tasks import agent_jobs as celery_agent_jobs
|
|
|
|
|
|
def _evaluator(params, start, end, commission, slippage):
|
|
fast = float(params["fast"])
|
|
slow = float(params["slow"])
|
|
quality = max(-1.0, 1.0 - abs(fast - 12.0) / 12.0 - abs(slow - 48.0) / 48.0)
|
|
days = max(2, (end.date() - start.date()).days + 1)
|
|
cost = (commission + slippage) * 1000.0
|
|
total_return = quality * 18.0 - cost
|
|
curve = []
|
|
equity = 10_000.0
|
|
for index in range(min(days, 90)):
|
|
base_return = (total_return / 100.0) / min(days, 90)
|
|
equity *= 1.0 + base_return * (0.85 + 0.15 * (index % 3))
|
|
curve.append({
|
|
"time": (start + timedelta(days=index)).isoformat(),
|
|
"value": equity,
|
|
"drawdown": min(0.0, -abs(index % 7 - 3) * 0.08),
|
|
})
|
|
return {
|
|
"totalReturn": total_return,
|
|
"sharpeRatio": quality * 2.0 - cost / 10.0,
|
|
"maxDrawdown": -max(2.0, (1.0 - quality) * 25.0 + cost),
|
|
"winRate": 55.0 + quality * 10.0,
|
|
"totalTrades": 16,
|
|
"equityCurve": curve,
|
|
}
|
|
|
|
|
|
def _parameters():
|
|
return [
|
|
SearchParameter("fast", "integer", 6, 18, 3, default=12),
|
|
SearchParameter("slow", "integer", 30, 66, 6, default=48),
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("method", ["random", "grid", "tpe"])
|
|
def test_engine_runs_all_search_methods_and_returns_v3_validation(method):
|
|
config = EvolutionConfig.from_payload({
|
|
"method": method,
|
|
"trials": 10,
|
|
"folds": 4,
|
|
"monteCarloPaths": 120,
|
|
"costMultipliers": [1, 2, 3],
|
|
"autoPrune": False,
|
|
})
|
|
result = StrategyEvolutionEngine(_evaluator).run(
|
|
parameters=_parameters(),
|
|
config=config,
|
|
start_date=datetime(2024, 1, 1),
|
|
end_date=datetime(2025, 12, 31),
|
|
commission=0.0005,
|
|
slippage=0.0005,
|
|
)
|
|
assert result["status"] == "complete"
|
|
assert len(result["trials"]) == 10
|
|
assert result["bestParams"]
|
|
assert result["heatmap"]["points"]
|
|
assert result["validation"]["pbo"]["samples"] > 0
|
|
assert result["validation"]["deflatedSharpe"]["observations"] > 0
|
|
assert result["validation"]["monteCarlo"]["paths"] == 120
|
|
assert [row["multiplier"] for row in result["validation"]["costStress"]] == [1.0, 2.0, 3.0]
|
|
|
|
|
|
def test_walk_forward_keeps_blind_holdout_after_validation():
|
|
plan = build_walk_forward_plan(
|
|
datetime(2024, 1, 1),
|
|
datetime(2025, 12, 31),
|
|
folds=5,
|
|
train_ratio=0.7,
|
|
blind_ratio=0.15,
|
|
embargo_days=2,
|
|
)
|
|
assert len(plan.folds) == 5
|
|
assert plan.blind_start is not None
|
|
assert all(fold.train_end < fold.validation_start <= fold.validation_end < plan.blind_start for fold in plan.folds)
|
|
assert all(left.validation_end < right.validation_start for left, right in zip(plan.folds, plan.folds[1:]))
|
|
assert all(left.train_end < right.train_end for left, right in zip(plan.folds, plan.folds[1:]))
|
|
|
|
|
|
def test_walk_forward_uses_intraday_observations_instead_of_calendar_days():
|
|
start = datetime(2026, 8, 27)
|
|
end = datetime(2026, 9, 25, 23, 59)
|
|
observations = tuple(pd.date_range(start, end, freq="1min").to_pydatetime())
|
|
plan = build_walk_forward_plan(
|
|
start,
|
|
end,
|
|
folds=4,
|
|
train_ratio=0.7,
|
|
blind_ratio=0.15,
|
|
embargo_bars=1,
|
|
warmup_bars=310,
|
|
observations=observations,
|
|
)
|
|
|
|
assert plan.sample_count == 43_200
|
|
assert plan.effective_folds == 4
|
|
assert plan.blind_bars == 6_480
|
|
assert plan.minimum_train_bars == 1_550
|
|
assert plan.minimum_validation_bars == 155
|
|
assert all(fold.train_bars >= 1_550 for fold in plan.folds)
|
|
assert all(fold.validation_bars >= 155 for fold in plan.folds)
|
|
assert plan.folds[-1].validation_end < plan.blind_start
|
|
|
|
|
|
def test_walk_forward_reduces_fold_count_when_bar_count_cannot_support_request():
|
|
start = datetime(2026, 1, 1)
|
|
observations = tuple(pd.date_range(start, periods=100, freq="1min").to_pydatetime())
|
|
plan = build_walk_forward_plan(
|
|
start,
|
|
observations[-1],
|
|
folds=4,
|
|
train_ratio=0.7,
|
|
blind_ratio=0,
|
|
embargo_bars=1,
|
|
warmup_bars=10,
|
|
observations=observations,
|
|
)
|
|
|
|
assert plan.requested_folds == 4
|
|
assert plan.effective_folds == 2
|
|
assert len(plan.folds) == 2
|
|
|
|
|
|
def test_walk_forward_rejects_only_when_market_observations_are_insufficient():
|
|
start = datetime(2026, 1, 1)
|
|
observations = tuple(pd.date_range(start, periods=700, freq="1min").to_pydatetime())
|
|
with pytest.raises(ValueError, match="strategyEvolution.observationRangeTooShort"):
|
|
build_walk_forward_plan(
|
|
start,
|
|
observations[-1],
|
|
folds=4,
|
|
train_ratio=0.7,
|
|
blind_ratio=0.15,
|
|
embargo_bars=1,
|
|
warmup_bars=310,
|
|
observations=observations,
|
|
)
|
|
|
|
|
|
def test_equity_returns_uses_daily_close_for_intraday_curves():
|
|
returns = equity_returns([
|
|
{"time": "2025-01-01T09:30:00", "value": 100.0},
|
|
{"time": "2025-01-01T16:00:00", "value": 102.0},
|
|
{"time": "2025-01-02T09:30:00", "value": 101.0},
|
|
{"time": "2025-01-02T16:00:00", "value": 103.0},
|
|
])
|
|
assert returns == pytest.approx([103.0 / 102.0 - 1.0])
|
|
|
|
|
|
def test_grid_sampler_is_bounded_and_deterministic():
|
|
first = ParameterSampler(_parameters(), seed=7).grid(7)
|
|
second = ParameterSampler(_parameters(), seed=7).grid(7)
|
|
assert first == second
|
|
assert len(first) == 7
|
|
assert len({(row["fast"], row["slow"]) for row in first}) == 7
|
|
|
|
|
|
def test_dual_moving_average_guard_becomes_parameter_constraint():
|
|
constraints = discover_parameter_constraints(
|
|
"""
|
|
def handle_data(context, data):
|
|
fast_period = int(context.params.get("fast_period", 20))
|
|
slow_period = int(context.params.get("slow_period", 60))
|
|
if fast_period >= slow_period:
|
|
return
|
|
""",
|
|
{"fast_period", "slow_period"},
|
|
)
|
|
assert [item.metadata() for item in constraints] == [
|
|
{"expression": "fast_period >= slow_period", "source": "guard"}
|
|
]
|
|
assert constraints[0].accepts({"fast_period": 20, "slow_period": 60}) is True
|
|
assert constraints[0].accepts({"fast_period": 83, "slow_period": 62}) is False
|
|
|
|
|
|
def test_generic_constraints_support_declarations_compound_guards_and_arithmetic():
|
|
constraints = discover_parameter_constraints(
|
|
"""
|
|
# @constraint stop_loss > 0 and take_profit >= stop_loss * 1.5
|
|
# @constraint mode in ("trend", "breakout")
|
|
def handle_data(context, data):
|
|
lookback = int(context.params.get("lookback", 20))
|
|
smoothing = int(context.params.get("smoothing", 5))
|
|
if smoothing <= 0 or lookback + smoothing > 100:
|
|
return
|
|
""",
|
|
{"stop_loss", "take_profit", "mode", "lookback", "smoothing"},
|
|
)
|
|
assert [item.metadata()["source"] for item in constraints] == ["declaration", "declaration", "guard"]
|
|
valid = {"stop_loss": 2, "take_profit": 4, "mode": "trend", "lookback": 20, "smoothing": 5}
|
|
assert all(item.accepts(valid) for item in constraints)
|
|
assert constraints[0].accepts({**valid, "take_profit": 2.5}) is False
|
|
assert constraints[1].accepts({**valid, "mode": "mean_reversion"}) is False
|
|
assert constraints[2].accepts({**valid, "lookback": 98}) is False
|
|
|
|
|
|
def test_constraint_rejections_and_flat_candidates_never_rank_as_best():
|
|
calls = []
|
|
|
|
def evaluator(params, start, end, commission, slippage):
|
|
calls.append((dict(params), start, end))
|
|
active = params["fast"] < params["slow"] and params["enabled"]
|
|
return {
|
|
"totalReturn": 3.0 if active else 0.0,
|
|
"sharpeRatio": 1.0 if active else 0.0,
|
|
"maxDrawdown": -2.0 if active else 0.0,
|
|
"winRate": 55.0 if active else 0.0,
|
|
"totalTrades": 4 if active else 0,
|
|
"totalExecutions": 8 if active else 0,
|
|
"equityCurve": [
|
|
{"time": start.isoformat(), "value": 10_000.0},
|
|
{"time": end.isoformat(), "value": 10_300.0 if active else 10_000.0},
|
|
],
|
|
}
|
|
|
|
parameters = [
|
|
SearchParameter("fast", "integer", 20, 80, 60, default=20),
|
|
SearchParameter("slow", "integer", 60, 60, 1, default=60),
|
|
SearchParameter("enabled", "boolean", default=True),
|
|
]
|
|
constraints = discover_parameter_constraints(
|
|
"""
|
|
def handle_data(context, data):
|
|
fast = int(context.params.get("fast", 20))
|
|
slow = int(context.params.get("slow", 60))
|
|
if fast >= slow:
|
|
return
|
|
""",
|
|
{"fast", "slow", "enabled"},
|
|
)
|
|
result = StrategyEvolutionEngine(evaluator).run(
|
|
parameters=parameters,
|
|
config=EvolutionConfig.from_payload({
|
|
"method": "grid",
|
|
"trials": 8,
|
|
"folds": 4,
|
|
"autoPrune": True,
|
|
"monteCarloPaths": 100,
|
|
"costMultipliers": [1],
|
|
}),
|
|
start_date=datetime(2024, 1, 1),
|
|
end_date=datetime(2025, 12, 31),
|
|
commission=0.0005,
|
|
slippage=0.0005,
|
|
constraints=constraints,
|
|
)
|
|
assert result["bestParams"] == {"fast": 20, "slow": 60, "enabled": True}
|
|
assert result["summary"]["constraintRejectedTrials"] == 2
|
|
assert all(item["components"]["validationActivity"] > 0 for item in result["topCandidates"])
|
|
assert len(calls) < len(result["trials"]) * 8
|
|
|
|
|
|
def test_non_finite_candidate_metrics_are_pruned_before_ranking():
|
|
def evaluator(params, start, end, commission, slippage):
|
|
valid = bool(params["enabled"])
|
|
return {
|
|
"totalReturn": 2.0 if valid else float("nan"),
|
|
"sharpeRatio": 0.8 if valid else float("inf"),
|
|
"maxDrawdown": -3.0,
|
|
"winRate": 50.0,
|
|
"totalTrades": 3,
|
|
"totalExecutions": 6,
|
|
"equityCurve": [
|
|
{"time": start.isoformat(), "value": 10_000.0},
|
|
{"time": end.isoformat(), "value": 10_200.0},
|
|
],
|
|
}
|
|
|
|
result = StrategyEvolutionEngine(evaluator).run(
|
|
parameters=[SearchParameter("enabled", "boolean", default=True)],
|
|
config=EvolutionConfig.from_payload({
|
|
"method": "grid",
|
|
"trials": 2,
|
|
"folds": 4,
|
|
"autoPrune": False,
|
|
"monteCarloPaths": 100,
|
|
"costMultipliers": [1],
|
|
}),
|
|
start_date=datetime(2024, 1, 1),
|
|
end_date=datetime(2025, 12, 31),
|
|
commission=0.0005,
|
|
slippage=0.0005,
|
|
)
|
|
invalid = next(item for item in result["trials"] if item["params"]["enabled"] is False)
|
|
assert invalid["pruned"] is True
|
|
assert invalid["reason"] == "invalidMetrics"
|
|
assert result["bestParams"] == {"enabled": True}
|
|
|
|
|
|
def test_all_inactive_candidates_report_structured_failure_diagnostics():
|
|
def evaluator(_params, start, end, _commission, _slippage):
|
|
return {
|
|
"totalReturn": 0.0,
|
|
"sharpeRatio": 0.0,
|
|
"maxDrawdown": 0.0,
|
|
"winRate": 0.0,
|
|
"totalTrades": 0,
|
|
"totalExecutions": 0,
|
|
"equityCurve": [
|
|
{"time": start.isoformat(), "value": 10_000.0},
|
|
{"time": end.isoformat(), "value": 10_000.0},
|
|
],
|
|
}
|
|
|
|
with pytest.raises(StrategyEvolutionFailure) as captured:
|
|
StrategyEvolutionEngine(evaluator).run(
|
|
parameters=[SearchParameter("enabled", "boolean", default=True)],
|
|
config=EvolutionConfig.from_payload({
|
|
"method": "grid",
|
|
"trials": 2,
|
|
"folds": 4,
|
|
"autoPrune": False,
|
|
"monteCarloPaths": 100,
|
|
"costMultipliers": [1],
|
|
}),
|
|
start_date=datetime(2024, 1, 1),
|
|
end_date=datetime(2025, 12, 31),
|
|
commission=0.0005,
|
|
slippage=0.0005,
|
|
)
|
|
|
|
failure = captured.value
|
|
assert str(failure) == "strategyEvolution.allTrialsPruned"
|
|
assert failure.details == {
|
|
"messageKey": "strategyEvolution.failure.noValidationActivity",
|
|
"dominantReason": "noActivity",
|
|
"totalTrials": 2,
|
|
"evaluatedTrials": 2,
|
|
"activeTrainingTrials": 0,
|
|
"activeValidationTrials": 0,
|
|
"actualBacktestRuns": None,
|
|
"reasonCounts": {"noActivity": 2},
|
|
}
|
|
|
|
|
|
def test_grid_estimate_uses_available_parameter_combinations():
|
|
estimate = StrategyEvolutionService.estimate({
|
|
"parameterSpace": [{"name": "enabled", "type": "boolean"}],
|
|
"config": {"method": "grid", "trials": 40, "folds": 4, "autoPrune": False, "costMultipliers": [1]},
|
|
})
|
|
assert estimate["trials"] == 2
|
|
assert estimate["requestedTrials"] == 40
|
|
assert estimate["backtestRuns"] == 4
|
|
assert estimate["isUpperBound"] is False
|
|
|
|
|
|
def test_tpe_estimate_keeps_requested_trials_when_range_changes():
|
|
estimate = StrategyEvolutionService.estimate({
|
|
"parameterSpace": [{"name": "period", "type": "integer", "min": 5, "max": 30, "step": 1}],
|
|
"config": {"method": "tpe", "trials": 24, "folds": 4, "autoPrune": True},
|
|
})
|
|
assert estimate["trials"] == 24
|
|
assert estimate["backtestRuns"] == 29
|
|
assert estimate["isUpperBound"] is True
|
|
|
|
|
|
def test_default_estimate_uses_interactive_trial_budget():
|
|
estimate = StrategyEvolutionService.estimate({
|
|
"parameterSpace": [{"name": "period", "type": "integer", "min": 5, "max": 30, "step": 1}],
|
|
"config": {"method": "tpe", "folds": 4, "autoPrune": True},
|
|
})
|
|
assert estimate["trials"] == 12
|
|
assert estimate["backtestRuns"] == 17
|
|
|
|
|
|
def test_prepared_evaluator_fetches_full_market_window_once():
|
|
calls = []
|
|
index = pd.date_range("2023-12-01", "2024-12-31", freq="1D")
|
|
frame = pd.DataFrame({
|
|
"open": range(100, 100 + len(index)),
|
|
"high": range(101, 101 + len(index)),
|
|
"low": range(99, 99 + len(index)),
|
|
"close": range(100, 100 + len(index)),
|
|
"volume": [1000.0] * len(index),
|
|
}, index=index)
|
|
|
|
def fetch(_market, _symbol, _timeframe, start_date, end_date, **_kwargs):
|
|
calls.append((start_date, end_date))
|
|
start = pd.Timestamp(start_date).tz_localize(None) if pd.Timestamp(start_date).tzinfo else pd.Timestamp(start_date)
|
|
end = pd.Timestamp(end_date).tz_localize(None) if pd.Timestamp(end_date).tzinfo else pd.Timestamp(end_date)
|
|
return frame.loc[(frame.index >= start) & (frame.index <= end)].copy()
|
|
|
|
code = """
|
|
def initialize(context):
|
|
context.set_universe(["USStock:AAPL"])
|
|
context.subscribe(frequency="1d")
|
|
context.set_warmup(2)
|
|
|
|
def handle_data(context, data):
|
|
pass
|
|
"""
|
|
evaluator = PreparedEvolutionEvaluator(
|
|
backtest_service=StrategyV2BacktestService(frame_fetcher=fetch),
|
|
user_id=7,
|
|
code=code,
|
|
start_date=datetime(2024, 1, 1),
|
|
end_date=datetime(2024, 12, 31),
|
|
initial_capital=10_000,
|
|
leverage_enabled=False,
|
|
leverage=1,
|
|
source_id=31,
|
|
strategy_name="Prepared",
|
|
)
|
|
assert len(calls) == 1
|
|
|
|
first = evaluator({"period": 10}, datetime(2024, 2, 1), datetime(2024, 4, 30), 0.0005, 0.0005)
|
|
second = evaluator({"period": 20}, datetime(2024, 5, 1), datetime(2024, 7, 31), 0.0005, 0.0005)
|
|
repeated = evaluator({"period": 20}, datetime(2024, 5, 1), datetime(2024, 7, 31), 0.0005, 0.0005)
|
|
|
|
assert len(calls) == 1
|
|
assert first["sampleCount"] > 0
|
|
assert second["sampleCount"] > 0
|
|
assert repeated is second
|
|
assert evaluator.stats == {"backtestRuns": 2, "cacheHits": 1}
|
|
|
|
context = evaluator.walk_forward_context(datetime(2024, 1, 1), datetime(2024, 12, 31))
|
|
assert context["frequency"] == "1d"
|
|
assert context["warmupBars"] == 2
|
|
assert len(context["observations"]) == 366
|
|
|
|
plan = build_walk_forward_plan(
|
|
datetime(2024, 1, 1),
|
|
datetime(2024, 12, 31),
|
|
folds=4,
|
|
train_ratio=0.7,
|
|
blind_ratio=0.15,
|
|
embargo_bars=2,
|
|
warmup_bars=context["warmupBars"],
|
|
observations=context["observations"],
|
|
)
|
|
segments = evaluator.evaluate_plan({"period": 30}, plan.folds, 0.0005, 0.0005)
|
|
assert len(segments) == 4
|
|
assert all(train["sampleCount"] > 0 and validation["sampleCount"] > 0 for train, validation in segments)
|
|
assert evaluator.stats["backtestRuns"] == 3
|
|
|
|
|
|
def test_robustness_statistics_are_bounded():
|
|
pbo = probability_of_backtest_overfitting([
|
|
[20, 21, 19, 18, 17, 16],
|
|
[10, 9, 11, 12, 10, 9],
|
|
[14, 13, 15, 12, 11, 13],
|
|
])
|
|
dsr = deflated_sharpe([0.01, -0.005, 0.008, 0.012, -0.002, 0.006], observed_sharpe=1.2, trials=20)
|
|
monte_carlo = monte_carlo_bootstrap([0.01, -0.005, 0.008, 0.012], paths=150, block_size=2, seed=5)
|
|
assert 0 <= pbo["probability"] <= 1
|
|
assert 0 <= dsr["probability"] <= 1
|
|
assert monte_carlo["paths"] == 150
|
|
assert sum(row["count"] for row in monte_carlo["terminalReturns"]) == 150
|
|
assert monte_carlo["uniqueTerminalReturns"] > 1
|
|
assert monte_carlo["effectiveBlockSize"] < monte_carlo["observations"]
|
|
|
|
|
|
def test_monte_carlo_short_series_does_not_collapse_to_circular_permutations():
|
|
result = monte_carlo_bootstrap(
|
|
[0.01, -0.02, 0.005, 0.03],
|
|
paths=1000,
|
|
block_size=5,
|
|
seed=42,
|
|
)
|
|
|
|
assert result["available"] is True
|
|
assert result["paths"] == 1000
|
|
assert result["observations"] == 4
|
|
assert result["effectiveBlockSize"] < result["observations"]
|
|
assert result["uniqueTerminalReturns"] > 1
|
|
assert len(result["terminalReturns"]) > 1
|
|
|
|
|
|
def test_parameter_contract_rejects_invalid_numeric_range():
|
|
with pytest.raises(ValueError, match="strategyEvolution.parameterRangeInvalid"):
|
|
SearchParameter.from_payload({"name": "period", "type": "integer", "min": 20, "max": 5})
|
|
|
|
|
|
def test_parameter_contract_accepts_canonical_text_values():
|
|
parameter = SearchParameter.from_payload({"name": "mode", "type": "text", "values": ["trend", "mean_reversion"]})
|
|
assert parameter.kind == "string"
|
|
assert parameter.choices == ("trend", "mean_reversion")
|
|
|
|
|
|
def test_parameter_space_recovers_declared_ranges_and_infers_missing_bounds():
|
|
contract = build_parameter_space({
|
|
"asset_type": "script",
|
|
"code": """
|
|
# @param fast_period int 20 range=2:100:1
|
|
# @param threshold float 0.2
|
|
# @param enabled bool true
|
|
def initialize(context):
|
|
pass
|
|
""",
|
|
"param_schema": {},
|
|
})
|
|
|
|
assert contract["strategyClass"] == "cta"
|
|
assert contract["scope"] == "declared"
|
|
by_name = {item["name"]: item for item in contract["parameters"]}
|
|
assert (by_name["fast_period"]["min"], by_name["fast_period"]["max"], by_name["fast_period"]["step"]) == (2, 100, 1)
|
|
assert by_name["threshold"]["min"] == pytest.approx(0.1)
|
|
assert by_name["threshold"]["max"] == pytest.approx(0.3)
|
|
assert by_name["enabled"]["choices"] == [False, True]
|
|
assert by_name["fast_period"]["recommendedMin"] == 10
|
|
assert by_name["fast_period"]["recommendedMax"] == 50
|
|
assert by_name["fast_period"]["defaultSelected"] is True
|
|
assert by_name["enabled"]["defaultSelected"] is False
|
|
assert by_name["enabled"]["optimizable"] is False
|
|
|
|
|
|
def test_boolean_strategy_modes_cannot_enter_the_evolution_search_space():
|
|
source = {
|
|
"param_schema": {
|
|
"params": [
|
|
{"name": "period", "type": "integer", "default": 20, "min": 5, "max": 100},
|
|
{"name": "allow_short", "type": "boolean", "default": True},
|
|
],
|
|
},
|
|
}
|
|
defaults = StrategyEvolutionService._parameters(source, None)
|
|
assert [item.name for item in defaults] == ["period"]
|
|
with pytest.raises(ValueError, match="strategyEvolution.parameterNotOptimizable"):
|
|
StrategyEvolutionService._parameters(source, [{"name": "allow_short", "type": "boolean"}])
|
|
|
|
|
|
def test_robot_parameter_space_exposes_only_runtime_bound_safe_controls():
|
|
source = {
|
|
"asset_type": "script",
|
|
"template_key": "robot_v2_dca",
|
|
"metadata": {"last_run_config": {"executor_type": "dca"}},
|
|
"param_schema": {},
|
|
"code": """
|
|
DCA_INTERVAL_MINUTES = 1440
|
|
DCA_MAX_ORDERS = 12
|
|
DCA_TOTAL_BUDGET_PCT = 0.8
|
|
DCA_ORDER_PCT = 0.1
|
|
DCA_PRICE_FILTER_ENABLED = True
|
|
DCA_MAX_ADVERSE_PRICE_PCT = 0.1
|
|
TAKE_PROFIT = 0.05
|
|
HARD_STOP = 0.1
|
|
PRICE_LEVELS = [0.9, 0.8]
|
|
def initialize(context):
|
|
pass
|
|
""",
|
|
}
|
|
|
|
contract = build_parameter_space(source)
|
|
assert contract["strategyClass"] == "robot"
|
|
assert contract["robotType"] == "dca"
|
|
assert contract["scope"] == "riskExecution"
|
|
names = [item["name"] for item in contract["parameters"]]
|
|
assert names == [
|
|
"dca_interval_minutes",
|
|
"dca_max_orders",
|
|
"dca_total_budget_pct",
|
|
"dca_order_pct",
|
|
"dca_price_filter_enabled",
|
|
"dca_max_adverse_price_pct",
|
|
"take_profit_pct",
|
|
"hard_stop_pct",
|
|
]
|
|
assert all("PRICE_LEVELS" != item.get("runtimeConstant") for item in contract["parameters"])
|
|
|
|
|
|
def test_robot_adapter_binds_candidate_params_before_original_initialize():
|
|
code = """
|
|
TAKE_PROFIT = 0.05
|
|
HARD_STOP = 0.1
|
|
def initialize(context):
|
|
context.set_universe(["USStock:AAPL"])
|
|
context.subscribe(frequency="1d")
|
|
|
|
def handle_data(context, data):
|
|
g.values = (TAKE_PROFIT, HARD_STOP)
|
|
"""
|
|
rows = [
|
|
{"name": "take_profit_pct", "type": "percent", "runtimeConstant": "TAKE_PROFIT"},
|
|
{"name": "hard_stop_pct", "type": "percent", "runtimeConstant": "HARD_STOP"},
|
|
]
|
|
adapted = adapt_source_for_parameters(code, rows)
|
|
StrategyV2BacktestService().compile(adapted)
|
|
namespace = {"g": type("State", (), {})()}
|
|
exec(adapted, namespace)
|
|
|
|
class Context:
|
|
params = {"take_profit_pct": 0.08, "hard_stop_pct": 0.12}
|
|
|
|
@staticmethod
|
|
def set_universe(_instruments):
|
|
return None
|
|
|
|
@staticmethod
|
|
def subscribe(**_kwargs):
|
|
return None
|
|
|
|
namespace["initialize"](Context())
|
|
namespace["handle_data"](Context(), None)
|
|
assert namespace["g"].values == pytest.approx((0.08, 0.12))
|
|
|
|
|
|
def test_parameter_request_can_narrow_but_not_expand_declared_range():
|
|
source = {"param_schema": {"params": [{"name": "period", "type": "integer", "min": 5, "max": 100, "step": 1}]}}
|
|
narrowed = StrategyEvolutionService._parameters(source, [{"name": "period", "min": 10, "max": 50, "type": "string"}])
|
|
assert narrowed[0].kind == "integer"
|
|
assert narrowed[0].minimum == 10
|
|
assert narrowed[0].maximum == 50
|
|
with pytest.raises(ValueError, match="strategyEvolution.parameterRangeOutsideDeclaration"):
|
|
StrategyEvolutionService._parameters(source, [{"name": "period", "min": 1, "max": 120}])
|
|
|
|
|
|
def test_parameter_request_defaults_to_the_server_generated_space_when_omitted():
|
|
source = {"param_schema": {"params": [{"name": "period", "type": "integer", "default": 20, "min": 5, "max": 100, "step": 1}]}}
|
|
parameters = StrategyEvolutionService._parameters(source, None)
|
|
assert len(parameters) == 1
|
|
assert parameters[0].name == "period"
|
|
assert parameters[0].minimum == 5
|
|
assert parameters[0].maximum == 100
|
|
|
|
|
|
def test_parameter_contract_compacts_regular_numeric_values_but_preserves_irregular_choices():
|
|
contract = build_parameter_space({
|
|
"param_schema": {
|
|
"params": [
|
|
{"name": "period", "type": "integer", "values": [5, 10, 15, 20]},
|
|
{"name": "threshold", "type": "number", "values": [0.1, 0.25, 0.9]},
|
|
],
|
|
},
|
|
})
|
|
regular, irregular = contract["parameters"]
|
|
assert regular["min"] == 5
|
|
assert regular["max"] == 20
|
|
assert regular["step"] == 5
|
|
assert "values" not in regular
|
|
assert "choices" not in regular
|
|
assert irregular["choices"] == [0.1, 0.25, 0.9]
|
|
assert "values" not in irregular
|
|
|
|
|
|
def test_service_runs_single_asset_cta_through_strategy_v2_backtester(monkeypatch):
|
|
source = {
|
|
"id": 31,
|
|
"name": "BTC CTA",
|
|
"asset_type": "script",
|
|
"code": "def initialize(context):\n context.set_universe(['Crypto:BTC/USDT@swap'])",
|
|
"param_schema": {
|
|
"params": [
|
|
{"name": "fast", "type": "integer", "min": 6, "max": 18, "step": 6},
|
|
{"name": "slow", "type": "integer", "min": 30, "max": 60, "step": 15},
|
|
],
|
|
},
|
|
}
|
|
|
|
class SourceService:
|
|
@staticmethod
|
|
def get_source(source_id, *, user_id):
|
|
assert source_id == 31
|
|
assert user_id == 7
|
|
return source
|
|
|
|
calls = []
|
|
|
|
class BacktestService:
|
|
@staticmethod
|
|
def run(**kwargs):
|
|
calls.append(kwargs)
|
|
return 1, _evaluator(
|
|
kwargs["params"],
|
|
kwargs["start_date"],
|
|
kwargs["end_date"],
|
|
kwargs["commission"],
|
|
kwargs["slippage"],
|
|
)
|
|
|
|
monkeypatch.setattr(evolution_service_module, "get_script_source_service", lambda: SourceService())
|
|
result = StrategyEvolutionService(backtest_service=BacktestService()).run(
|
|
user_id=7,
|
|
payload={
|
|
"sourceId": 31,
|
|
"startDate": "2024-01-01",
|
|
"endDate": "2025-12-31",
|
|
"commission": 0.0005,
|
|
"slippage": 0.0005,
|
|
"parameterSpace": source["param_schema"]["params"],
|
|
"config": {
|
|
"method": "grid",
|
|
"trials": 4,
|
|
"folds": 4,
|
|
"autoPrune": False,
|
|
"monteCarloPaths": 100,
|
|
"costMultipliers": [1],
|
|
},
|
|
},
|
|
)
|
|
|
|
assert result["status"] == "complete"
|
|
assert result["source"] == {"id": 31, "name": "BTC CTA"}
|
|
assert calls
|
|
assert all(call["source_id"] == 31 for call in calls)
|
|
assert all(call["code"] == source["code"] for call in calls)
|
|
assert all(set(call["params"]) == {"fast", "slow"} for call in calls)
|
|
|
|
|
|
def test_grid_engine_does_not_repeat_exhausted_combinations():
|
|
result = StrategyEvolutionEngine(lambda params, start, end, commission, slippage: _evaluator({"fast": 12, "slow": 48}, start, end, commission, slippage)).run(
|
|
parameters=[SearchParameter("enabled", "boolean", default=True)],
|
|
config=EvolutionConfig.from_payload({"method": "grid", "trials": 40, "folds": 4, "autoPrune": False, "monteCarloPaths": 100}),
|
|
start_date=datetime(2024, 1, 1),
|
|
end_date=datetime(2025, 12, 31),
|
|
commission=0.0005,
|
|
slippage=0.0005,
|
|
)
|
|
assert len(result["trials"]) == 2
|
|
|
|
|
|
def test_background_runner_keeps_user_scope_out_of_engine_payload(monkeypatch):
|
|
captured = {}
|
|
|
|
class Service:
|
|
@staticmethod
|
|
def run(*, user_id, payload, on_progress):
|
|
captured.update(user_id=user_id, payload=payload, on_progress=on_progress)
|
|
return {"status": "complete"}
|
|
|
|
monkeypatch.setattr(evolution_routes, "get_strategy_evolution_service", lambda: Service())
|
|
progress = lambda event: event
|
|
result = evolution_routes._run_evolution_job({"__userId": 17, "sourceId": 9}, progress)
|
|
assert result == {"status": "complete"}
|
|
assert captured == {"user_id": 17, "payload": {"sourceId": 9}, "on_progress": progress}
|
|
|
|
|
|
def test_celery_dispatches_strategy_evolution_to_durable_runner(monkeypatch):
|
|
captured = {}
|
|
|
|
def runner(payload, on_progress):
|
|
captured.update(payload=payload, on_progress=on_progress)
|
|
return {"status": "complete"}
|
|
|
|
monkeypatch.setattr(evolution_routes, "_run_evolution_job", runner)
|
|
progress = lambda event: event
|
|
|
|
assert celery_agent_jobs.supports_kind("strategy_evolution") is True
|
|
result = celery_agent_jobs._execute(
|
|
"strategy_evolution",
|
|
{"__userId": 17, "sourceId": 9},
|
|
progress,
|
|
)
|
|
|
|
assert result == {"status": "complete"}
|
|
assert captured == {
|
|
"payload": {"__userId": 17, "sourceId": 9},
|
|
"on_progress": progress,
|
|
}
|
|
|
|
|
|
def test_celery_persists_structured_evolution_failure(monkeypatch):
|
|
from app.utils import agent_jobs
|
|
|
|
details = {
|
|
"messageKey": "strategyEvolution.failure.noValidationActivity",
|
|
"dominantReason": "noActivity",
|
|
"totalTrials": 12,
|
|
"reasonCounts": {"noActivity": 12},
|
|
}
|
|
events = []
|
|
monkeypatch.setattr(agent_jobs, "get_job_for_worker", lambda _job_id: {
|
|
"status": "queued",
|
|
"kind": "strategy_evolution",
|
|
"request": {"sourceId": 189},
|
|
})
|
|
monkeypatch.setattr(agent_jobs, "_set_status", lambda *_args, **_kwargs: True)
|
|
monkeypatch.setattr(agent_jobs, "_set_failure", lambda *_args, **_kwargs: True)
|
|
monkeypatch.setattr(agent_jobs, "_publish_progress", lambda _job_id, event, **_kwargs: events.append(event))
|
|
monkeypatch.setattr(
|
|
celery_agent_jobs,
|
|
"_execute",
|
|
lambda *_args, **_kwargs: (_ for _ in ()).throw(
|
|
StrategyEvolutionFailure("strategyEvolution.allTrialsPruned", details)
|
|
),
|
|
)
|
|
|
|
with pytest.raises(StrategyEvolutionFailure):
|
|
celery_agent_jobs.execute_agent_job.run("evolution-job")
|
|
|
|
assert events[-1]["phase"] == "failed"
|
|
assert events[-1]["error"] == "strategyEvolution.allTrialsPruned"
|
|
assert events[-1]["failure"] == details
|
|
|
|
|
|
def test_public_job_decodes_persisted_json():
|
|
output = evolution_routes._public_job({
|
|
"job_id": "job-1",
|
|
"status": "succeeded",
|
|
"result": '{"status":"complete"}',
|
|
"progress": '{"completed":4,"total":4}',
|
|
"request": '{"sourceId":9,"startDate":"2025-01-01","__userId":17}',
|
|
})
|
|
assert output["jobId"] == "job-1"
|
|
assert output["result"]["status"] == "complete"
|
|
assert output["progress"]["completed"] == 4
|
|
assert output["request"] == {"sourceId": 9, "startDate": "2025-01-01"}
|
|
|
|
|
|
def test_history_endpoint_lists_only_selected_strategy_evolution_jobs(monkeypatch):
|
|
captured = {}
|
|
|
|
def list_jobs(**kwargs):
|
|
captured.update(kwargs)
|
|
return [{"job_id": "job-1"}]
|
|
|
|
def get_job(job_id, *, user_id):
|
|
assert job_id == "job-1"
|
|
assert user_id == 23
|
|
return {
|
|
"job_id": job_id,
|
|
"kind": "strategy_evolution",
|
|
"status": "running",
|
|
"request": {"sourceId": 7, "__userId": 23},
|
|
"progress": {"completed": 2, "total": 8},
|
|
}
|
|
|
|
monkeypatch.setattr(evolution_routes.agent_jobs, "list_jobs", list_jobs)
|
|
monkeypatch.setattr(evolution_routes.agent_jobs, "get_job", get_job)
|
|
app = Flask(__name__)
|
|
with app.test_request_context("/?limit=10&sourceId=7"):
|
|
g.user_id = 23
|
|
response = inspect.unwrap(evolution_routes.list_strategy_evolution_jobs)()
|
|
|
|
payload = response.get_json()
|
|
assert captured == {
|
|
"user_id": 23,
|
|
"kind": "strategy_evolution",
|
|
"request_source_id": 7,
|
|
"limit": 10,
|
|
}
|
|
assert payload["data"]["items"][0]["request"] == {"sourceId": 7}
|
|
|
|
|
|
def test_history_detects_only_running_jobs_past_the_worker_time_limit(monkeypatch):
|
|
monkeypatch.setenv("CELERY_TASK_TIME_LIMIT", "3600")
|
|
stale = datetime.now(timezone.utc) - timedelta(seconds=3901)
|
|
recent = datetime.now(timezone.utc) - timedelta(seconds=3899)
|
|
assert evolution_routes._is_stale_running_job({"status": "running", "started_at": stale}) is True
|
|
assert evolution_routes._is_stale_running_job({"status": "running", "started_at": recent}) is False
|
|
assert evolution_routes._is_stale_running_job({"status": "succeeded", "started_at": stale}) is False
|