{
  "expected_cases": 400,
  "result_files": 400,
  "scored_cases": 399,
  "failed_cases": [
    {
      "task_id": "5995e97dbf4293a612ab56eb",
      "question_id": "Q024",
      "model": "google/gemini-3.8-flash",
      "method": "sql_streaming",
      "error": "Model requested tools during forced finalization"
    }
  ],
  "missing_cases": [],
  "unexpected_cases": [],
  "duplicate_cases": [],
  "api_attempts": 3441,
  "api_successful_attempts": 3437,
  "failed_http_attempts": [
    {
      "tag": "19121a41f3331f7d46ef7dfd:agent:2",
      "call_id": "7da216bc183c4d929a33356864cfba88",
      "attempt": 1,
      "status_code": 500,
      "error": "HTTP 500: {'error': {'code': 'application_error', 'message': 'HTTP 599: Timeout while connecting', 'type': 'application_error'}}",
      "usage": {},
      "requested_model": "google/gemini-3.8-flash",
      "requested_provider": "google-ai-studio",
      "request_file": "raw/7da216bc183c4d929a33356864cfba88.request.json",
      "response_file": "raw/7da216bc183c4d929a33356864cfba88.response.json",
      "retry_recovered": true
    },
    {
      "tag": "29cac7c7d2935f62d9e8e358:judge",
      "call_id": "e1087fcce4584d8387d70e936758cf70",
      "attempt": 1,
      "status_code": 429,
      "error": "HTTP 429: {'error': {'message': 'Provider returned error', 'code': 429, 'metadata': {'raw': 'anthropic/claude-haiku-5.5 is temporarily rate-limited upstream. Please retry shortly, or add your own key to accumulate your rate limits: https://openrouter.ai/settings/integrations', 'provider_name': 'Anthropic', 'is_byok': False, 'limit_source': 'upstream_provider_shared_pool', 'remedy_hint': 'Retry shortly, add your own provider key (https://openrouter.ai/settings/integrations), or route to another provider with provider routing: https://openrouter.ai/docs/features/provider-routing', 'retry_after_seconds': 1, 'retry_after_seconds_raw': 1, 'headers': {'Retry-After': '1'}}}, 'user_id': 'org_2v7amPvBH6U161qxuAATjm39Zg3'}",
      "usage": {},
      "requested_model": "anthropic/claude-haiku-5.5",
      "requested_provider": "anthropic",
      "request_file": "raw/e1087fcce4584d8387d70e936758cf70.request.json",
      "response_file": "raw/e1087fcce4584d8387d70e936758cf70.response.json",
      "retry_recovered": true
    },
    {
      "tag": "c73207874e8bd9143c6fa8cd:judge",
      "call_id": "ca8785ba50fe43b7bb701cf938d1bb76",
      "attempt": 1,
      "status_code": 429,
      "error": "HTTP 429: {'error': {'message': 'Provider returned error', 'code': 429, 'metadata': {'raw': 'anthropic/claude-haiku-5.5 is temporarily rate-limited upstream. Please retry shortly, or add your own key to accumulate your rate limits: https://openrouter.ai/settings/integrations', 'provider_name': 'Anthropic', 'is_byok': False, 'limit_source': 'upstream_provider_shared_pool', 'remedy_hint': 'Retry shortly, add your own provider key (https://openrouter.ai/settings/integrations), or route to another provider with provider routing: https://openrouter.ai/docs/features/provider-routing', 'retry_after_seconds': 1, 'retry_after_seconds_raw': 1, 'headers': {'Retry-After': '1'}}}, 'user_id': 'org_2v7amPvBH6U161qxuAATjm39Zg3'}",
      "usage": {},
      "requested_model": "anthropic/claude-haiku-5.5",
      "requested_provider": "anthropic",
      "request_file": "raw/ca8785ba50fe43b7bb701cf938d1bb76.request.json",
      "response_file": "raw/ca8785ba50fe43b7bb701cf938d1bb76.response.json",
      "retry_recovered": true
    },
    {
      "tag": "223c156cffee35cf4d88f201:judge",
      "call_id": "953d792313c24b39896b449b07c01840",
      "attempt": 1,
      "status_code": 429,
      "error": "HTTP 429: {'error': {'message': 'Provider returned error', 'code': 429, 'metadata': {'raw': 'anthropic/claude-haiku-5.5 is temporarily rate-limited upstream. Please retry shortly, or add your own key to accumulate your rate limits: https://openrouter.ai/settings/integrations', 'provider_name': 'Anthropic', 'is_byok': False, 'limit_source': 'upstream_provider_shared_pool', 'remedy_hint': 'Retry shortly, add your own provider key (https://openrouter.ai/settings/integrations), or route to another provider with provider routing: https://openrouter.ai/docs/features/provider-routing', 'retry_after_seconds': 1, 'retry_after_seconds_raw': 1, 'headers': {'Retry-After': '1'}}}, 'user_id': 'org_2v7amPvBH6U161qxuAATjm39Zg3'}",
      "usage": {},
      "requested_model": "anthropic/claude-haiku-5.5",
      "requested_provider": "anthropic",
      "request_file": "raw/953d792313c24b39896b449b07c01840.request.json",
      "response_file": "raw/953d792313c24b39896b449b07c01840.response.json",
      "retry_recovered": true
    }
  ],
  "metadata_http_attempts": 4,
  "metadata_http_failures": 0,
  "inference_requests_without_audit": [],
  "raw_checks": {
    "attempts_checked": 3441,
    "usage_matches_passed": 3441,
    "requested_model_matches_passed": 3441,
    "low_reasoning_passed": 3441,
    "provider_fallback_disabled_passed": 3441,
    "provider_only_matches_passed": 3441,
    "zero_temperature_when_sent_passed": 1639,
    "reported_provider_matched_passed": 3437,
    "actual_model_matches_raw_passed": 3437,
    "actual_provider_matches_raw_passed": 3437,
    "response_id_matches_raw_passed": 3437,
    "temperature_capability_respected_passed": 3441,
    "agent_initial_inputs_checked": 400,
    "question_only_initial_input_passed": 400
  },
  "raw_integrity_problems": [],
  "question_input_mismatches": [],
  "case_structure_problems": [],
  "judge_input_or_score_problems": [],
  "summary_recalculation_differences": [],
  "billing_recalculation_matches": true,
  "billing": {
    "known_usd": 7.950589285,
    "unknown_attempts": 4,
    "complete": false,
    "preflight_known_usd": 0.0001661
  },
  "resource": {
    "peak_memory_mib": 2048.0,
    "max_swap_bytes": 0,
    "memory_events": {
      "low": 0,
      "high": 0,
      "max": 3261,
      "oom": 0,
      "oom_kill": 0,
      "oom_group_kill": 0,
      "sock_throttled": 0
    },
    "wall_seconds": 3450.5451586300005
  },
  "recorded_sources_unchanged": true,
  "recorded_code_unchanged": true,
  "tool_calls_requested": 2715,
  "tool_outputs": 2714,
  "tool_errors": 311,
  "tool_error_categories": {
    "memory_limit_or_allocation": 86,
    "sql_schema_or_syntax": 132,
    "duplicate_filing_id": 80,
    "tool_budget": 6,
    "invalid_tool_arguments": 5,
    "other": 2
  },
  "most_common_tool_errors": [
    {
      "error": "Duplicate filing ID in source",
      "count": 50
    },
    {
      "error": "Binder Error: Referenced column \"parsed_md\" was not found because the FROM clause is missing",
      "count": 33
    },
    {
      "error": "Binder Error: Referenced column \"company_name\" not found in FROM clause!",
      "count": 30
    },
    {
      "error": "Duplicate filing accession in source",
      "count": 30
    },
    {
      "error": "Binder Error: Referenced column \"file_id\" not found in FROM clause!",
      "count": 23
    },
    {
      "error": "malloc of size 646993472 failed",
      "count": 22
    },
    {
      "error": "Binder Error: Referenced column \"form\" not found in FROM clause!",
      "count": 20
    },
    {
      "error": "Binder Error: Referenced column \"form_type\" not found in FROM clause!",
      "count": 11
    },
    {
      "error": "Tool budget exhausted",
      "count": 6
    },
    {
      "error": "Out of Memory Error: failed to allocate data of size 128.0 MiB (598.2 KiB/122.0 MiB used)",
      "count": 4
    },
    {
      "error": "Out of Memory Error: failed to allocate data of size 128.0 MiB (568.1 KiB/122.0 MiB used)",
      "count": 4
    },
    {
      "error": "Out of Memory Error: failed to allocate data of size 1.0 GiB (328.0 KiB/122.0 MiB used)",
      "count": 4
    },
    {
      "error": "Out of Memory Error: failed to allocate data of size 128.0 MiB (480.2 KiB/122.0 MiB used)",
      "count": 4
    },
    {
      "error": "Out of Memory Error: failed to allocate data of size 128.0 MiB (476.2 KiB/122.0 MiB used)",
      "count": 3
    },
    {
      "error": "Out of Memory Error: failed to allocate data of size 128.0 MiB (594.2 KiB/122.0 MiB used)",
      "count": 3
    },
    {
      "error": "Binder Error: Referenced column \"filing_date\" not found in FROM clause!",
      "count": 3
    },
    {
      "error": "Binder Error: No function matches the given name and argument types 'strpos(VARCHAR, STRING_LITERAL, INTEGER_LITERAL)'. You might need to add explicit type casts.",
      "count": 3
    },
    {
      "error": "query() got an unexpected keyword argument 'file_id'",
      "count": 3
    },
    {
      "error": "Out of Memory Error: failed to allocate data of size 128.0 MiB (380.2 KiB/122.0 MiB used)",
      "count": 3
    },
    {
      "error": "Out of Memory Error: failed to allocate data of size 128.0 MiB (384.2 KiB/122.0 MiB used)",
      "count": 2
    }
  ],
  "cases_with_tool_errors": 190,
  "cases_reaching_tool_budget": 199,
  "cache_hits": 524,
  "heuristic_review_cases": [
    {
      "task_id": "1c6a46f541939e707a4985d5",
      "question_id": "Q053",
      "model": "openai/gpt-6-luna",
      "method": "sql",
      "correctness": 95,
      "note": "Heuristic review only: short/structured SQL evidence or appropriate abstention can be valid. Not proof of leakage."
    },
    {
      "task_id": "1d6afc06d6f4782ce86a7e02",
      "question_id": "Q059",
      "model": "google/gemini-3.8-flash",
      "method": "sql",
      "correctness": 98,
      "note": "Heuristic review only: short/structured SQL evidence or appropriate abstention can be valid. Not proof of leakage."
    },
    {
      "task_id": "1e05261f01bc125c7ee85ec4",
      "question_id": "Q076",
      "model": "google/gemini-3.8-flash",
      "method": "sql",
      "correctness": 95,
      "note": "Heuristic review only: short/structured SQL evidence or appropriate abstention can be valid. Not proof of leakage."
    },
    {
      "task_id": "3b25696a0678b185a0d264c6",
      "question_id": "Q076",
      "model": "openai/gpt-6-luna",
      "method": "sql_streaming",
      "correctness": 92,
      "note": "Heuristic review only: short/structured SQL evidence or appropriate abstention can be valid. Not proof of leakage."
    },
    {
      "task_id": "3e580c187c15e50815598e3a",
      "question_id": "Q076",
      "model": "openai/gpt-6-luna",
      "method": "sql",
      "correctness": 90,
      "note": "Heuristic review only: short/structured SQL evidence or appropriate abstention can be valid. Not proof of leakage."
    },
    {
      "task_id": "8baf5e91b5d8b2dd85c4b7a6",
      "question_id": "Q053",
      "model": "openai/gpt-6-luna",
      "method": "sql_streaming",
      "correctness": 100,
      "note": "Heuristic review only: short/structured SQL evidence or appropriate abstention can be valid. Not proof of leakage."
    },
    {
      "task_id": "ac147b05f8da93840204ecf0",
      "question_id": "Q020",
      "model": "google/gemini-3.8-flash",
      "method": "sql",
      "correctness": 90,
      "note": "Heuristic review only: short/structured SQL evidence or appropriate abstention can be valid. Not proof of leakage."
    },
    {
      "task_id": "b3b6f095d584e82ad5802fff",
      "question_id": "Q085",
      "model": "google/gemini-3.8-flash",
      "method": "sql_streaming",
      "correctness": 90,
      "note": "Heuristic review only: short/structured SQL evidence or appropriate abstention can be valid. Not proof of leakage."
    },
    {
      "task_id": "bb4eb29fc94dded8a7ecfe08",
      "question_id": "Q083",
      "model": "openai/gpt-6-luna",
      "method": "sql_streaming",
      "correctness": 100,
      "note": "Heuristic review only: short/structured SQL evidence or appropriate abstention can be valid. Not proof of leakage."
    },
    {
      "task_id": "cd8e4c9882b720412ff64aa2",
      "question_id": "Q059",
      "model": "openai/gpt-6-luna",
      "method": "sql",
      "correctness": 95,
      "note": "Heuristic review only: short/structured SQL evidence or appropriate abstention can be valid. Not proof of leakage."
    },
    {
      "task_id": "f67cef7319817ef6179e509f",
      "question_id": "Q059",
      "model": "google/gemini-3.8-flash",
      "method": "sql_streaming",
      "correctness": 97,
      "note": "Heuristic review only: short/structured SQL evidence or appropriate abstention can be valid. Not proof of leakage."
    }
  ],
  "findings": [
    {
      "severity": "error",
      "code": "case_completeness",
      "title": "399/400 cases have final answers and Haiku scores",
      "detail": "Q024 / Gemini Flash / sql_streaming returned a tool call during forced finalization and was not judged."
    },
    {
      "severity": "warning",
      "code": "http_retries",
      "title": "4 failed HTTP attempts are retained",
      "detail": "Failed HTTP attempts were followed by later successful replies for their tags. This differs from the terminal case-level failure."
    },
    {
      "severity": "warning",
      "code": "billing",
      "title": "Billing contains unpriced failed attempts",
      "detail": "Known reported charges are a subtotal. Missing usage on HTTP errors is unknown, not confirmed zero."
    },
    {
      "severity": "warning",
      "code": "tool_retrieval_errors",
      "title": "311 tool error responses",
      "detail": "SQL schema/syntax errors, duplicate filing-ID rejection and bounded allocation failures can consume the eight-tool budget and cause abstention. A scored case is not proof of successful retrieval."
    },
    {
      "severity": "warning",
      "code": "duplicate_ids",
      "title": "176333 duplicated filing-ID groups inside source files",
      "detail": "Scoped SQL and filing-read tools reject duplicate IDs. Next experiments should distinguish identical stored-checksum copies from conflicting records and support an explicit row identity. No originals were deduplicated or altered."
    },
    {
      "severity": "warning",
      "code": "memory",
      "title": "Memory reached 2048 MiB; 0 OOM kills",
      "detail": "The process-tree limit prevented machine-wide growth. Reaching memory.max still caused pressure and some worker allocation failures; no swap was used."
    },
    {
      "severity": "info",
      "code": "score_denominator",
      "title": "Scores are judge estimates over scored cases",
      "detail": "The primary mean excludes the failed case. A separate coverage-adjusted mean treats missing scores as zero without changing the original score. Citation support and correctness are distinct; neither is a guarantee of objective truth."
    },
    {
      "severity": "info",
      "code": "latency_fairness",
      "title": "Shared-cache, concurrent-run latency",
      "detail": "Four concurrent cases, one serial data tool and a shared cache mean queueing and cache order affect timing. This is not an independent cold-cache latency comparison. The same 100 questions repeat across four conditions, so do not treat 400 scores as independent questions."
    },
    {
      "severity": "info",
      "code": "reference_limit",
      "title": "Reference and evidence limitations",
      "detail": "Judge inputs and scores were mechanically checked against raw payloads. Gold quotes were checked for visibility in retrieved outputs, not reverified against every full source filing. Expected-filing recall only counts explicit accession IDs in structured outputs. Long-excerpt flags are heuristics, not proof of leakage."
    },
    {
      "severity": "warning",
      "code": "question_supplied_operands",
      "title": "Some questions already supply the financial operands",
      "detail": "Q053, Q059, Q076, Q083 and Q085 include numbers sufficient for substantial arithmetic answers. High correctness with little retrieved text can therefore reflect calculation from the question, rather than successful document retrieval. For example, Q059 / Flash / SQL used a constant arithmetic SELECT; Haiku awarded 98 correctness and 80 citation support. Report correctness and retrieval indicators separately."
    },
    {
      "severity": "pass",
      "code": "consistency",
      "title": "Saved payloads, scores and summaries reconcile",
      "detail": "See structured audit checks and the original-preservation manifest. No requests were retried and no answers were rejudged."
    }
  ],
  "mechanically_consistent": true
}