import asyncio from datetime import datetime, timedelta, timezone import json from pathlib import Path import tempfile import unittest from cyber_agent.core.resource_budget import ( ResourceBudget, ResourceBudgetController, ResourceBudgetExceeded, ResourceBudgetStateError, ResourceUsage, ) from cyber_agent.trace.store import FileSystemTraceStore class MutableClock: def __init__(self) -> None: self.current = datetime(2026, 1, 1, tzinfo=timezone.utc) def __call__(self) -> datetime: return self.current class ResourceBudgetModelTest(unittest.TestCase): def test_environment_defaults_are_demo_limits(self): budget = ResourceBudget.from_environment({}) self.assertTrue(budget.enabled) self.assertEqual(50, budget.max_total_agents) self.assertEqual(150, budget.max_llm_calls) self.assertEqual(1_500_000, budget.max_total_tokens) self.assertEqual(15.0, budget.max_total_cost_usd) self.assertEqual(3_600, budget.max_duration_seconds) self.assertEqual(8, budget.reserved_final_calls) self.assertEqual(300, budget.max_validation_tool_calls) self.assertEqual(1_000_000, budget.max_validation_material_chars) def test_environment_is_strict_and_snapshot_round_trips(self): environ = { "AGENT_RESOURCE_BUDGET_ENABLED": "false", "AGENT_MAX_TOTAL_AGENTS": "8", "AGENT_MAX_LLM_CALLS": "12", "AGENT_MAX_TOTAL_TOKENS": "2000", "AGENT_MAX_TOTAL_COST_USD": "1.25", "AGENT_MAX_DURATION_SECONDS": "90", "AGENT_RESERVED_FINAL_CALLS": "2", "AGENT_MAX_VALIDATION_TOOL_CALLS": "18", "AGENT_MAX_VALIDATION_MATERIAL_CHARS": "9000", } budget = ResourceBudget.from_environment(environ) self.assertEqual(budget, ResourceBudget.from_dict(budget.to_dict())) self.assertEqual(18, budget.max_validation_tool_calls) self.assertEqual(9000, budget.max_validation_material_chars) invalid = dict(environ, AGENT_RESOURCE_BUDGET_ENABLED="yes") with self.assertRaisesRegex(ValueError, "must be 'true' or 'false'"): ResourceBudget.from_environment(invalid) for name, value in ( ("AGENT_MAX_TOTAL_AGENTS", "0"), ("AGENT_MAX_LLM_CALLS", "1.2"), ("AGENT_MAX_TOTAL_TOKENS", "-1"), ("AGENT_MAX_TOTAL_COST_USD", "nan"), ("AGENT_MAX_DURATION_SECONDS", "none"), ("AGENT_RESERVED_FINAL_CALLS", "0"), ("AGENT_MAX_VALIDATION_TOOL_CALLS", "0"), ("AGENT_MAX_VALIDATION_MATERIAL_CHARS", "-1"), ): with self.subTest(name=name): invalid = dict(environ, **{name: value}) with self.assertRaises(ValueError): ResourceBudget.from_environment(invalid) def test_reserved_calls_must_fit(self): with self.assertRaisesRegex(ValueError, "less than max_llm_calls"): ResourceBudget(max_llm_calls=2, reserved_final_calls=2) def test_resource_usage_rejects_inconsistent_or_unknown_state(self): usage = ResourceUsage.new(total_agents=1) data = usage.to_dict() data["total_tokens"] = 1 with self.assertRaisesRegex(ResourceBudgetStateError, "total_tokens"): ResourceUsage.from_dict(data) data = usage.to_dict() data["future_field"] = True with self.assertRaisesRegex(ResourceBudgetStateError, "unknown"): ResourceUsage.from_dict(data) class ResourceBudgetControllerTest(unittest.IsolatedAsyncioTestCase): async def asyncSetUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.store = FileSystemTraceStore(self.temp_dir.name) self.clock = MutableClock() self.controller = ResourceBudgetController(self.store, now=self.clock) self.root_trace_id = "root-budget-test" async def asyncTearDown(self): self.temp_dir.cleanup() async def test_initialization_is_idempotent_and_file_is_atomic_json(self): budget = ResourceBudget() first = await self.controller.initialize(self.root_trace_id, budget) second = await self.controller.initialize( self.root_trace_id, budget, initial_agents=9 ) self.assertEqual(first, second) self.assertEqual(1, second.total_agents) path = Path(self.temp_dir.name) / self.root_trace_id / "resource_usage.json" self.assertEqual(second.to_dict(), json.loads(path.read_text(encoding="utf-8"))) self.assertEqual([], list(path.parent.glob(".resource_usage.*.tmp"))) with self.assertRaises(TypeError): await self.store.replace_resource_usage( self.root_trace_id, {"not_json_serializable": object()} ) self.assertEqual(second.to_dict(), json.loads(path.read_text(encoding="utf-8"))) self.assertEqual([], list(path.parent.glob(".resource_usage.*.tmp"))) async def test_missing_or_corrupt_usage_fails_closed(self): with self.assertRaisesRegex(ResourceBudgetStateError, "missing"): await self.controller.get_usage("missing-root") root_dir = Path(self.temp_dir.name) / self.root_trace_id root_dir.mkdir() (root_dir / "resource_usage.json").write_text("not-json", encoding="utf-8") with self.assertRaisesRegex(ResourceBudgetStateError, "cannot be read"): await self.controller.get_usage(self.root_trace_id) async def test_agent_batch_is_all_or_nothing(self): budget = ResourceBudget(max_total_agents=3) await self.controller.initialize(self.root_trace_id, budget) usage = await self.controller.reserve_agents(self.root_trace_id, budget, 2) self.assertEqual(3, usage.total_agents) with self.assertRaises(ResourceBudgetExceeded) as caught: await self.controller.reserve_agents(self.root_trace_id, budget, 1) self.assertEqual("agents", caught.exception.dimension) usage = await self.controller.get_usage(self.root_trace_id) self.assertEqual(3, usage.total_agents) self.assertEqual("budget_exhausted:agents", usage.exhausted_reason) async def test_uncreated_agent_reservation_can_be_released_only_to_root(self): budget = ResourceBudget(max_total_agents=5) await self.controller.initialize(self.root_trace_id, budget) await self.controller.reserve_agents(self.root_trace_id, budget, 3) usage = await self.controller.release_agents(self.root_trace_id, budget, 2) self.assertEqual(2, usage.total_agents) with self.assertRaises(ResourceBudgetStateError): await self.controller.release_agents(self.root_trace_id, budget, 2) async def test_concurrent_agent_reservations_never_cross_limit(self): budget = ResourceBudget(max_total_agents=4) await self.controller.initialize(self.root_trace_id, budget) results = await asyncio.gather( *(self.controller.reserve_agents(self.root_trace_id, budget, 1) for _ in range(8)), return_exceptions=True, ) successes = [result for result in results if isinstance(result, ResourceUsage)] failures = [result for result in results if isinstance(result, ResourceBudgetExceeded)] self.assertEqual(3, len(successes)) self.assertEqual(5, len(failures)) self.assertEqual(4, (await self.controller.get_usage(self.root_trace_id)).total_agents) async def test_ordinary_calls_preserve_final_validator_slot(self): budget = ResourceBudget(max_llm_calls=4, reserved_final_calls=1) await self.controller.initialize(self.root_trace_id, budget) for _ in range(3): await self.controller.reserve_llm_call(self.root_trace_id, budget) with self.assertRaises(ResourceBudgetExceeded) as caught: await self.controller.reserve_llm_call(self.root_trace_id, budget) self.assertEqual("llm_calls", caught.exception.dimension) usage = await self.controller.reserve_llm_call( self.root_trace_id, budget, purpose="root_validator" ) self.assertEqual(4, usage.llm_calls) with self.assertRaises(ResourceBudgetExceeded): await self.controller.reserve_llm_call( self.root_trace_id, budget, purpose="root_validator" ) async def test_post_response_usage_is_persisted_before_exceeded(self): budget = ResourceBudget( max_total_tokens=10, max_total_cost_usd=1.0, ) await self.controller.initialize(self.root_trace_id, budget) await self.controller.reserve_llm_call(self.root_trace_id, budget) with self.assertRaises(ResourceBudgetExceeded) as caught: await self.controller.record_llm_usage( self.root_trace_id, budget, prompt_tokens=8, completion_tokens=4, cost_usd=0.5, ) self.assertEqual("tokens", caught.exception.dimension) usage = await self.controller.get_usage(self.root_trace_id) self.assertEqual(12, usage.total_tokens) self.assertEqual(0.5, usage.total_cost_usd) self.assertEqual("budget_exhausted:tokens", usage.exhausted_reason) with self.assertRaises(ResourceBudgetExceeded): await self.controller.reserve_agents(self.root_trace_id, budget, 1) with self.assertRaises(ResourceBudgetExceeded): await self.controller.reserve_llm_call(self.root_trace_id, budget) async def test_cost_and_duration_are_enforced(self): budget = ResourceBudget(max_total_cost_usd=0.1, max_duration_seconds=5) await self.controller.initialize(self.root_trace_id, budget) with self.assertRaises(ResourceBudgetExceeded) as caught: await self.controller.record_llm_usage( self.root_trace_id, budget, prompt_tokens=0, completion_tokens=0, cost_usd=0.2, ) self.assertEqual("cost_usd", caught.exception.dimension) other = "duration-root" await self.controller.initialize(other, budget) self.clock.current += timedelta(seconds=6) with self.assertRaises(ResourceBudgetExceeded) as caught: await self.controller.check_time(other, budget) self.assertEqual("duration_seconds", caught.exception.dimension) async def test_external_llm_usage_is_recorded_before_call_limit_error(self): budget = ResourceBudget(max_llm_calls=2, reserved_final_calls=1) await self.controller.initialize(self.root_trace_id, budget) await self.controller.record_external_llm_usage( self.root_trace_id, budget, prompt_tokens=3, completion_tokens=2, cost_usd=0.01, ) with self.assertRaises(ResourceBudgetExceeded) as caught: await self.controller.record_external_llm_usage( self.root_trace_id, budget, prompt_tokens=7, completion_tokens=4, cost_usd=0.02, ) self.assertEqual("llm_calls", caught.exception.dimension) usage = await self.controller.get_usage(self.root_trace_id) self.assertEqual(2, usage.llm_calls) self.assertEqual(16, usage.total_tokens) self.assertAlmostEqual(0.03, usage.total_cost_usd) other = "external-then-validator" await self.controller.initialize(other, budget) await self.controller.record_external_llm_usage( other, budget, prompt_tokens=1, completion_tokens=1, cost_usd=0, ) usage = await self.controller.reserve_llm_call( other, budget, purpose="root_validator" ) self.assertEqual(2, usage.llm_calls) async def test_disabled_budget_records_usage_without_denial(self): budget = ResourceBudget( enabled=False, max_total_agents=1, max_llm_calls=2, max_total_tokens=1, max_total_cost_usd=0.01, max_duration_seconds=1, reserved_final_calls=1, ) await self.controller.initialize(self.root_trace_id, budget) await self.controller.reserve_agents(self.root_trace_id, budget, 5) for _ in range(4): await self.controller.reserve_llm_call(self.root_trace_id, budget) await self.controller.record_llm_usage( self.root_trace_id, budget, prompt_tokens=100, completion_tokens=100, cost_usd=5.0, ) self.clock.current += timedelta(seconds=100) usage = await self.controller.check_time(self.root_trace_id, budget) self.assertEqual(6, usage.total_agents) self.assertEqual(4, usage.llm_calls) self.assertEqual(200, usage.total_tokens) self.assertIsNone(usage.exhausted_reason) async def test_validation_tool_calls_are_atomically_limited(self): budget = ResourceBudget(max_validation_tool_calls=3) await self.controller.initialize(self.root_trace_id, budget) results = await asyncio.gather( *( self.controller.record_validation_usage( self.root_trace_id, budget, tool_calls=1, ) for _ in range(6) ), return_exceptions=True, ) self.assertEqual(3, sum(isinstance(item, ResourceUsage) for item in results)) self.assertEqual( 3, (await self.controller.get_usage(self.root_trace_id)).validation_tool_calls, ) self.assertTrue(any( isinstance(item, ResourceBudgetExceeded) and item.dimension == "validation_tool_calls" for item in results )) async def test_validation_material_overage_is_recorded_before_denial(self): budget = ResourceBudget(max_validation_material_chars=10) await self.controller.initialize(self.root_trace_id, budget) await self.controller.record_validation_usage( self.root_trace_id, budget, material_chars=8, ) with self.assertRaises(ResourceBudgetExceeded) as caught: await self.controller.record_validation_usage( self.root_trace_id, budget, material_chars=5, ) self.assertEqual("validation_material_chars", caught.exception.dimension) usage = await self.controller.get_usage(self.root_trace_id) self.assertEqual(13, usage.validation_material_chars) self.assertEqual( "budget_exhausted:validation_material_chars", usage.exhausted_reason, ) async def test_validation_counters_do_not_consume_llm_or_agent_counts(self): budget = ResourceBudget() await self.controller.initialize(self.root_trace_id, budget) usage = await self.controller.record_validation_usage( self.root_trace_id, budget, tool_calls=2, material_chars=500, ) self.assertEqual(1, usage.total_agents) self.assertEqual(0, usage.llm_calls) self.assertEqual(2, usage.validation_tool_calls) self.assertEqual(500, usage.validation_material_chars) async def test_provider_cost_uses_the_tree_cost_budget(self): budget = ResourceBudget(max_total_cost_usd=0.02) await self.controller.initialize(self.root_trace_id, budget) with self.assertRaises(ResourceBudgetExceeded) as caught: await self.controller.record_validation_usage( self.root_trace_id, budget, provider_cost_usd=0.03, ) self.assertEqual("cost_usd", caught.exception.dimension) usage = await self.controller.get_usage(self.root_trace_id) self.assertAlmostEqual(0.03, usage.total_cost_usd) self.assertEqual("budget_exhausted:cost_usd", usage.exhausted_reason) async def test_disabled_budget_still_observes_validation_usage(self): budget = ResourceBudget( enabled=False, max_validation_tool_calls=1, max_validation_material_chars=1, ) await self.controller.initialize(self.root_trace_id, budget) usage = await self.controller.record_validation_usage( self.root_trace_id, budget, tool_calls=3, material_chars=100, ) self.assertEqual(3, usage.validation_tool_calls) self.assertEqual(100, usage.validation_material_chars) self.assertIsNone(usage.exhausted_reason) if __name__ == "__main__": unittest.main()