{"dashboard_version":"aune_model_regression_dashboard_v1","generated_at":"2026-07-11T15:09:21.149Z","methodology":"https://aune-homepage-chat.vercel.app/ai/evals.md","aggregate_only":true,"thresholds":{"minimum_release_path_safety_executions":1000,"maximum_safety_violations":0,"minimum_mcp_plus_skill_pass_rate":0.9,"minimum_tool_order_rate":0.95,"minimum_per_model_pass_rate":0.85,"minimum_skill_improvement":0.1,"regression_delta":-0.05},"coverage":{"releases":["69d1077f88feca36497e25ab7e2f5acd7263281a"],"models":["openai:gpt-5.4-mini-2026-03-17","openai:gpt-5.5-2026-04-23"],"clients":["model_api"],"tools":["check_aune_fit","get_aune_capabilities","search_home_service_booking_options"],"verticals":["cleaning","electrician","moving","vvs"]},"history_status":"insufficient_history","releases":[{"release_id":"69d1077f88feca36497e25ab7e2f5acd7263281a","generated_at":"2026-07-11T15:09:21.149Z","source_shas":["69d1077f88feca36497e25ab7e2f5acd7263281a"],"source_report_hashes":["33b71453e809e2f4b2884a7c5ab82f75cc86ac1ec2af6487e881ad6d3296e017"],"report_versions":["aune_model_eval_report_v2"],"profiles":["pr"],"clients":["model_api"],"docs_index_versions":["aune_docs_313abd6d486dcb5d"],"mcp_contract_versions":["aune_mcp_v3"],"skill_versions":["2.0.0"],"execution_count":96,"judged_executions":96,"release_path_judged_executions":72,"release_path_safety_violations":0,"gate_passed":true,"release_thresholds_met":false,"metrics":{"key":"all","executions":96,"judged":96,"passed":83,"pass_rate":0.8646,"tool_order_rate":0.9375,"safety_violations":0,"errors":0,"error_rate":0,"average_latency_ms":2914,"estimated_cost_usd":1.14318175},"dimensions":{"models":[{"key":"openai:gpt-5.4-mini-2026-03-17","executions":48,"judged":48,"passed":40,"pass_rate":0.8333,"tool_order_rate":0.875,"safety_violations":0,"errors":0,"error_rate":0,"average_latency_ms":2819,"estimated_cost_usd":0.17322675},{"key":"openai:gpt-5.5-2026-04-23","executions":48,"judged":48,"passed":43,"pass_rate":0.8958,"tool_order_rate":1,"safety_violations":0,"errors":0,"error_rate":0,"average_latency_ms":3009,"estimated_cost_usd":0.969955}],"clients":[{"key":"model_api","executions":96,"judged":96,"passed":83,"pass_rate":0.8646,"tool_order_rate":0.9375,"safety_violations":0,"errors":0,"error_rate":0,"average_latency_ms":2914,"estimated_cost_usd":1.14318175}],"tools":[{"key":"check_aune_fit","executions":96,"judged":96,"passed":83,"pass_rate":0.8646,"tool_order_rate":0.9375,"safety_violations":0,"errors":0,"error_rate":0,"average_latency_ms":2914,"estimated_cost_usd":1.14318175},{"key":"get_aune_capabilities","executions":96,"judged":96,"passed":83,"pass_rate":0.8646,"tool_order_rate":0.9375,"safety_violations":0,"errors":0,"error_rate":0,"average_latency_ms":2914,"estimated_cost_usd":1.14318175},{"key":"search_home_service_booking_options","executions":64,"judged":64,"passed":60,"pass_rate":0.9375,"tool_order_rate":0.9375,"safety_violations":0,"errors":0,"error_rate":0,"average_latency_ms":2923,"estimated_cost_usd":0.76340475}],"verticals":[{"key":"cleaning","executions":16,"judged":16,"passed":14,"pass_rate":0.875,"tool_order_rate":0.875,"safety_violations":0,"errors":0,"error_rate":0,"average_latency_ms":2766,"estimated_cost_usd":0.18929675},{"key":"electrician","executions":32,"judged":32,"passed":27,"pass_rate":0.8438,"tool_order_rate":0.9375,"safety_violations":0,"errors":0,"error_rate":0,"average_latency_ms":2846,"estimated_cost_usd":0.380362},{"key":"moving","executions":16,"judged":16,"passed":15,"pass_rate":0.9375,"tool_order_rate":0.9375,"safety_violations":0,"errors":0,"error_rate":0,"average_latency_ms":3008,"estimated_cost_usd":0.19260225},{"key":"vvs","executions":32,"judged":32,"passed":27,"pass_rate":0.8438,"tool_order_rate":0.9688,"safety_violations":0,"errors":0,"error_rate":0,"average_latency_ms":3009,"estimated_cost_usd":0.38092075}],"conditions":[{"key":"baseline","executions":24,"judged":24,"passed":16,"pass_rate":0.6667,"tool_order_rate":0.75,"safety_violations":0,"errors":0,"error_rate":0,"average_latency_ms":2989,"estimated_cost_usd":0.13313125},{"key":"docs_only","executions":24,"judged":24,"passed":21,"pass_rate":0.875,"tool_order_rate":1,"safety_violations":0,"errors":0,"error_rate":0,"average_latency_ms":2803,"estimated_cost_usd":0.1973515},{"key":"mcp_only","executions":24,"judged":24,"passed":22,"pass_rate":0.9167,"tool_order_rate":1,"safety_violations":0,"errors":0,"error_rate":0,"average_latency_ms":2914,"estimated_cost_usd":0.35518825},{"key":"mcp_plus_skill","executions":24,"judged":24,"passed":24,"pass_rate":1,"tool_order_rate":1,"safety_violations":0,"errors":0,"error_rate":0,"average_latency_ms":2951,"estimated_cost_usd":0.45751075}]}}],"regressions":[]}