{"schema_version":4,"scorecard":{"note":"Computed by scripts/scorecard.py's functions from the saved runs: the same numbers as the README's Results section.","order":{"models":"pass rate after feedback (the second print), highest first, then lower total cost (scorecard.rows())","test_first_order":"test-first pass rate on the four tasks, highest first, then lower cost per pass (scorecard.tested_order(); the README's ranking and its picture use this order)","not_tested_order":"first-print pass rate, highest first"},"tasks":{"tripod-plate":{"label":"Tripod plate","short":"Tripod"},"bearing-wheel":{"label":"Closet door wheel","short":"Wheel"},"nut-knob/M3":{"label":"Hand knob, M3","short":"Knob M3"},"nut-knob/M5":{"label":"Hand knob, M5","short":"Knob M5"},"estoa-light":{"label":"Estoa light box (harder)","short":"Estoa box"}},"scored_tasks":["tripod-plate","bearing-wheel","nut-knob/M3","nut-knob/M5"],"harder_task":"estoa-light","min_trials":5,"skipped":"Grok 4.7 was left out because it was too expensive to run 10 trials on.","models":[{"model":"google/gemini-4-argon","short":"Gemini 4 Argon","replayed":true,"first":{"by_task":{"tripod-plate":{"passes":3,"trials":10,"not_graded":0},"bearing-wheel":{"passes":7,"trials":10,"not_graded":0},"nut-knob/M3":{"passes":1,"trials":10,"not_graded":4},"nut-knob/M5":{"passes":9,"trials":10,"not_graded":0}},"passes":20,"trials":40,"rate":0.5,"not_graded":4,"excluded":{"count":0,"by_reason":[],"by_task":{}}},"after_feedback":{"by_task":{"tripod-plate":{"passes":10,"trials":10,"not_graded":0,"passed_first":3,"passed_after_feedback":7},"bearing-wheel":{"passes":7,"trials":10,"not_graded":0,"passed_first":7,"passed_after_feedback":0},"nut-knob/M3":{"passes":5,"trials":10,"not_graded":0,"passed_first":1,"passed_after_feedback":4},"nut-knob/M5":{"passes":10,"trials":10,"not_graded":0,"passed_first":9,"passed_after_feedback":1}},"passes":32,"trials":40,"passed_first":20,"passed_after_feedback":12,"rate":0.8,"not_graded":0,"replays_replaced_by_regrade":14,"cost_usd":6.160332,"cost_per_pass":0.192510375,"est_print_s_per_pass":1549.09375,"est_grams_per_pass":13.803749999999999},"test_first":{"by_task":{"tripod-plate":{"passes":7,"trials":10,"not_graded":0,"cant_tell":{"trials":9,"passes":7}},"bearing-wheel":{"passes":6,"trials":8,"not_graded":1,"cant_tell":{"trials":7,"passes":5}},"nut-knob/M3":{"passes":5,"trials":5,"not_graded":0,"cant_tell":{"trials":4,"passes":4}},"nut-knob/M5":{"passes":5,"trials":5,"not_graded":0,"cant_tell":{"trials":5,"passes":5}},"estoa-light":{"passes":0,"trials":3,"not_graded":0,"cant_tell":{"trials":0,"passes":0}}},"passes":23,"trials":28,"rate":0.8214285714285714,"cost_per_pass":0.19008252173913043,"est_print_s_per_pass":1401.4782608695652,"est_grams_per_pass":13.181304347826087,"cant_tell":{"trials":25,"passes":21},"excluded":{"count":12,"by_reason":[{"reason":"infrastructure error, not a result: out of OpenRouter credit (HTTP 402)","count":12}],"by_task":{"bearing-wheel":2,"nut-knob/M3":5,"nut-knob/M5":5}}},"tokens_per_trial":20520.525,"change_points":32},{"model":"anthropic/claude-opus-5.5","short":"Opus 5.5","replayed":true,"first":{"by_task":{"tripod-plate":{"passes":1,"trials":10,"not_graded":0},"bearing-wheel":{"passes":1,"trials":10,"not_graded":2},"nut-knob/M3":{"passes":4,"trials":10,"not_graded":1},"nut-knob/M5":{"passes":10,"trials":10,"not_graded":0}},"passes":16,"trials":40,"rate":0.4,"not_graded":3,"excluded":{"count":0,"by_reason":[],"by_task":{}}},"after_feedback":{"by_task":{"tripod-plate":{"passes":7,"trials":10,"not_graded":0,"passed_first":1,"passed_after_feedback":6},"bearing-wheel":{"passes":6,"trials":10,"not_graded":2,"passed_first":1,"passed_after_feedback":5},"nut-knob/M3":{"passes":4,"trials":10,"not_graded":3,"passed_first":4,"passed_after_feedback":0},"nut-knob/M5":{"passes":10,"trials":10,"not_graded":0,"passed_first":10,"passed_after_feedback":0}},"passes":27,"trials":40,"passed_first":16,"passed_after_feedback":11,"rate":0.675,"not_graded":5,"replays_replaced_by_regrade":6,"cost_usd":9.722254,"cost_per_pass":0.36008348148148145,"est_print_s_per_pass":2327.703703703704,"est_grams_per_pass":22.098518518518517},"test_first":{"by_task":{"tripod-plate":{"passes":3,"trials":10,"not_graded":1,"cant_tell":{"trials":1,"passes":0}},"bearing-wheel":{"passes":4,"trials":10,"not_graded":0,"cant_tell":{"trials":9,"passes":4}},"nut-knob/M3":{"passes":5,"trials":10,"not_graded":3,"cant_tell":{"trials":8,"passes":3}},"nut-knob/M5":{"passes":9,"trials":10,"not_graded":0,"cant_tell":{"trials":9,"passes":8}},"estoa-light":{"passes":0,"trials":3,"not_graded":1,"cant_tell":{"trials":0,"passes":0}}},"passes":21,"trials":40,"rate":0.525,"cost_per_pass":0.2874130476190476,"est_print_s_per_pass":1868.047619047619,"est_grams_per_pass":16.755714285714287,"cant_tell":{"trials":27,"passes":15},"excluded":{"count":0,"by_reason":[],"by_task":{}}},"tokens_per_trial":21911.525,"change_points":12},{"model":"google/gemini-3.8-flash","short":"Gemini 3.8 Flash","replayed":true,"first":{"by_task":{"tripod-plate":{"passes":4,"trials":10,"not_graded":0},"bearing-wheel":{"passes":4,"trials":10,"not_graded":0},"nut-knob/M3":{"passes":1,"trials":10,"not_graded":3},"nut-knob/M5":{"passes":9,"trials":10,"not_graded":0}},"passes":18,"trials":40,"rate":0.45,"not_graded":3,"excluded":{"count":0,"by_reason":[],"by_task":{}}},"after_feedback":{"by_task":{"tripod-plate":{"passes":6,"trials":10,"not_graded":0,"passed_first":4,"passed_after_feedback":2},"bearing-wheel":{"passes":4,"trials":10,"not_graded":0,"passed_first":4,"passed_after_feedback":0},"nut-knob/M3":{"passes":5,"trials":10,"not_graded":1,"passed_first":1,"passed_after_feedback":4},"nut-knob/M5":{"passes":10,"trials":10,"not_graded":0,"passed_first":9,"passed_after_feedback":1}},"passes":25,"trials":40,"passed_first":18,"passed_after_feedback":7,"rate":0.625,"not_graded":1,"replays_replaced_by_regrade":7,"cost_usd":6.225419,"cost_per_pass":0.24901675999999998,"est_print_s_per_pass":1602.36,"est_grams_per_pass":14.7728},"test_first":{"by_task":{"tripod-plate":{"passes":2,"trials":10,"not_graded":0,"cant_tell":{"trials":8,"passes":1}},"bearing-wheel":{"passes":5,"trials":5,"not_graded":0,"cant_tell":{"trials":0,"passes":0}},"nut-knob/M3":{"passes":3,"trials":5,"not_graded":0,"cant_tell":{"trials":3,"passes":1}},"nut-knob/M5":{"passes":4,"trials":5,"not_graded":0,"cant_tell":{"trials":5,"passes":4}},"estoa-light":{"passes":0,"trials":3,"not_graded":1,"cant_tell":{"trials":0,"passes":0}}},"passes":14,"trials":25,"rate":0.56,"cost_per_pass":0.2610387857142857,"est_print_s_per_pass":1399.4285714285713,"est_grams_per_pass":14.124285714285715,"cant_tell":{"trials":16,"passes":6},"excluded":{"count":15,"by_reason":[{"reason":"infrastructure error, not a result: out of OpenRouter credit (HTTP 402)","count":15}],"by_task":{"bearing-wheel":5,"nut-knob/M3":5,"nut-knob/M5":5}}},"tokens_per_trial":51339.55,"change_points":11},{"model":"deepseek/deepseek-v4.1-flash","short":"DeepSeek V4.1 Flash","replayed":true,"first":{"by_task":{"tripod-plate":{"passes":1,"trials":10,"not_graded":0},"bearing-wheel":{"passes":1,"trials":10,"not_graded":0},"nut-knob/M3":{"passes":5,"trials":10,"not_graded":4},"nut-knob/M5":{"passes":9,"trials":11,"not_graded":1}},"passes":16,"trials":41,"rate":0.3902439024390244,"not_graded":5,"excluded":{"count":0,"by_reason":[],"by_task":{}}},"after_feedback":{"by_task":{"tripod-plate":{"passes":4,"trials":10,"not_graded":1,"passed_first":1,"passed_after_feedback":3},"bearing-wheel":{"passes":6,"trials":10,"not_graded":0,"passed_first":1,"passed_after_feedback":5},"nut-knob/M3":{"passes":5,"trials":10,"not_graded":3,"passed_first":5,"passed_after_feedback":0},"nut-knob/M5":{"passes":10,"trials":11,"not_graded":0,"passed_first":9,"passed_after_feedback":1}},"passes":25,"trials":41,"passed_first":16,"passed_after_feedback":9,"rate":0.6097560975609756,"not_graded":4,"replays_replaced_by_regrade":8,"cost_usd":2.325528,"cost_per_pass":0.09302112,"est_print_s_per_pass":3142.16,"est_grams_per_pass":29.1312},"test_first":{"by_task":{"tripod-plate":{"passes":3,"trials":9,"not_graded":0,"cant_tell":{"trials":9,"passes":3}},"bearing-wheel":{"passes":2,"trials":5,"not_graded":0,"cant_tell":{"trials":4,"passes":2}},"nut-knob/M3":{"passes":1,"trials":5,"not_graded":2,"cant_tell":{"trials":4,"passes":1}},"nut-knob/M5":{"passes":4,"trials":7,"not_graded":0,"cant_tell":{"trials":6,"passes":3}},"estoa-light":{"passes":1,"trials":3,"not_graded":0,"cant_tell":{"trials":1,"passes":0}}},"passes":10,"trials":26,"rate":0.38461538461538464,"cost_per_pass":0.1556856,"est_print_s_per_pass":3725.2,"est_grams_per_pass":35.477,"cant_tell":{"trials":23,"passes":9},"excluded":{"count":14,"by_reason":[{"reason":"infrastructure error, not a result: out of OpenRouter credit (HTTP 402)","count":14}],"by_task":{"bearing-wheel":5,"nut-knob/M3":5,"nut-knob/M5":3,"tripod-plate":1}}},"tokens_per_trial":171988.70731707316,"change_points":-1},{"model":"openai/gpt-6.1-sol","short":"GPT-6.1 Sol","replayed":true,"first":{"by_task":{"tripod-plate":{"passes":1,"trials":10,"not_graded":0},"bearing-wheel":{"passes":8,"trials":10,"not_graded":0},"nut-knob/M3":{"passes":2,"trials":10,"not_graded":4},"nut-knob/M5":{"passes":9,"trials":10,"not_graded":1}},"passes":20,"trials":40,"rate":0.5,"not_graded":5,"excluded":{"count":0,"by_reason":[],"by_task":{}}},"after_feedback":{"by_task":{"tripod-plate":{"passes":1,"trials":10,"not_graded":0,"passed_first":1,"passed_after_feedback":0},"bearing-wheel":{"passes":9,"trials":10,"not_graded":0,"passed_first":8,"passed_after_feedback":1},"nut-knob/M3":{"passes":2,"trials":10,"not_graded":4,"passed_first":2,"passed_after_feedback":0},"nut-knob/M5":{"passes":9,"trials":10,"not_graded":1,"passed_first":9,"passed_after_feedback":0}},"passes":21,"trials":40,"passed_first":20,"passed_after_feedback":1,"rate":0.525,"not_graded":5,"replays_replaced_by_regrade":11,"cost_usd":3.53515,"cost_per_pass":0.16834047619047618,"est_print_s_per_pass":2336.5714285714284,"est_grams_per_pass":19.350952380952382},"test_first":{"by_task":{"tripod-plate":{"passes":2,"trials":10,"not_graded":0,"cant_tell":{"trials":10,"passes":2}},"bearing-wheel":{"passes":7,"trials":10,"not_graded":2,"cant_tell":{"trials":4,"passes":1}},"nut-knob/M3":{"passes":6,"trials":10,"not_graded":0,"cant_tell":{"trials":6,"passes":2}},"nut-knob/M5":{"passes":10,"trials":10,"not_graded":0,"cant_tell":{"trials":10,"passes":10}},"estoa-light":{"passes":2,"trials":3,"not_graded":1,"cant_tell":{"trials":3,"passes":2}}},"passes":25,"trials":40,"rate":0.625,"cost_per_pass":0.07926332,"est_print_s_per_pass":1902.0,"est_grams_per_pass":17.6332,"cant_tell":{"trials":30,"passes":15},"excluded":{"count":0,"by_reason":[],"by_task":{}}},"tokens_per_trial":27146.325,"change_points":12},{"model":"anthropic/claude-sonnet-5.5","short":"Sonnet 5.5","replayed":true,"first":{"by_task":{"tripod-plate":{"passes":0,"trials":10,"not_graded":0},"bearing-wheel":{"passes":0,"trials":10,"not_graded":0},"nut-knob/M3":{"passes":5,"trials":10,"not_graded":2},"nut-knob/M5":{"passes":9,"trials":10,"not_graded":1}},"passes":14,"trials":40,"rate":0.35,"not_graded":3,"excluded":{"count":0,"by_reason":[],"by_task":{}}},"after_feedback":{"by_task":{"tripod-plate":{"passes":2,"trials":10,"not_graded":0,"passed_first":0,"passed_after_feedback":2},"bearing-wheel":{"passes":2,"trials":10,"not_graded":2,"passed_first":0,"passed_after_feedback":2},"nut-knob/M3":{"passes":6,"trials":10,"not_graded":2,"passed_first":5,"passed_after_feedback":1},"nut-knob/M5":{"passes":10,"trials":10,"not_graded":0,"passed_first":9,"passed_after_feedback":1}},"passes":20,"trials":40,"passed_first":14,"passed_after_feedback":6,"rate":0.5,"not_graded":4,"replays_replaced_by_regrade":5,"cost_usd":5.889107,"cost_per_pass":0.29445535,"est_print_s_per_pass":3129.3,"est_grams_per_pass":30.601499999999998},"test_first":{"by_task":{"tripod-plate":{"passes":1,"trials":10,"not_graded":0,"cant_tell":{"trials":10,"passes":1}},"bearing-wheel":{"passes":0,"trials":10,"not_graded":0,"cant_tell":{"trials":6,"passes":0}},"nut-knob/M3":{"passes":9,"trials":10,"not_graded":0,"cant_tell":{"trials":3,"passes":2}},"nut-knob/M5":{"passes":10,"trials":10,"not_graded":0,"cant_tell":{"trials":7,"passes":7}},"estoa-light":{"passes":1,"trials":3,"not_graded":0,"cant_tell":{"trials":3,"passes":1}}},"passes":20,"trials":40,"rate":0.5,"cost_per_pass":0.2093679,"est_print_s_per_pass":2385.8,"est_grams_per_pass":22.9775,"cant_tell":{"trials":26,"passes":10},"excluded":{"count":0,"by_reason":[],"by_task":{}}},"tokens_per_trial":25299.175,"change_points":15},{"model":"z-ai/glm-5.3-flash","short":"GLM 5.3 Flash","replayed":false,"first":{"by_task":{"tripod-plate":{"passes":0,"trials":10,"not_graded":0},"bearing-wheel":{"passes":0,"trials":10,"not_graded":1},"nut-knob/M3":{"passes":4,"trials":10,"not_graded":1},"nut-knob/M5":{"passes":5,"trials":10,"not_graded":0}},"passes":9,"trials":40,"rate":0.225,"not_graded":2,"excluded":{"count":0,"by_reason":[],"by_task":{}}},"after_feedback":{"by_task":{"tripod-plate":{"passes":0,"trials":10,"not_graded":0,"passed_first":0,"passed_after_feedback":0},"bearing-wheel":{"passes":0,"trials":10,"not_graded":1,"passed_first":0,"passed_after_feedback":0},"nut-knob/M3":{"passes":4,"trials":10,"not_graded":1,"passed_first":4,"passed_after_feedback":0},"nut-knob/M5":{"passes":5,"trials":10,"not_graded":0,"passed_first":5,"passed_after_feedback":0}},"passes":9,"trials":40,"passed_first":9,"passed_after_feedback":0,"rate":0.225,"not_graded":2,"replays_replaced_by_regrade":0,"cost_usd":1.485409,"cost_per_pass":0.16504544444444444,"est_print_s_per_pass":3157.0,"est_grams_per_pass":28.61555555555556},"test_first":null,"tokens_per_trial":245938.3,"change_points":null},{"model":"openai/gpt-6-luna","short":"GPT-6 Luna","replayed":false,"first":{"by_task":{"tripod-plate":{"passes":0,"trials":10,"not_graded":0},"bearing-wheel":{"passes":0,"trials":10,"not_graded":0},"nut-knob/M3":{"passes":0,"trials":10,"not_graded":1},"nut-knob/M5":{"passes":3,"trials":10,"not_graded":0}},"passes":3,"trials":40,"rate":0.075,"not_graded":1,"excluded":{"count":0,"by_reason":[],"by_task":{}}},"after_feedback":{"by_task":{"tripod-plate":{"passes":0,"trials":10,"not_graded":0,"passed_first":0,"passed_after_feedback":0},"bearing-wheel":{"passes":0,"trials":10,"not_graded":0,"passed_first":0,"passed_after_feedback":0},"nut-knob/M3":{"passes":0,"trials":10,"not_graded":1,"passed_first":0,"passed_after_feedback":0},"nut-knob/M5":{"passes":3,"trials":10,"not_graded":0,"passed_first":3,"passed_after_feedback":0}},"passes":3,"trials":40,"passed_first":3,"passed_after_feedback":0,"rate":0.075,"not_graded":1,"replays_replaced_by_regrade":0,"cost_usd":0.180157,"cost_per_pass":0.06005233333333334,"est_print_s_per_pass":9239.333333333334,"est_grams_per_pass":90.61},"test_first":null,"tokens_per_trial":25682.075,"change_points":null},{"model":"deepseek/deepseek-v4-pro-0813","short":"DeepSeek V4 Pro","replayed":false,"first":{"by_task":{"tripod-plate":{"passes":0,"trials":10,"not_graded":2},"bearing-wheel":{"passes":1,"trials":10,"not_graded":0},"nut-knob/M3":{"passes":1,"trials":10,"not_graded":2},"nut-knob/M5":{"passes":0,"trials":10,"not_graded":0}},"passes":2,"trials":40,"rate":0.05,"not_graded":4,"excluded":{"count":0,"by_reason":[],"by_task":{}}},"after_feedback":{"by_task":{"tripod-plate":{"passes":0,"trials":10,"not_graded":2,"passed_first":0,"passed_after_feedback":0},"bearing-wheel":{"passes":1,"trials":10,"not_graded":0,"passed_first":1,"passed_after_feedback":0},"nut-knob/M3":{"passes":1,"trials":10,"not_graded":2,"passed_first":1,"passed_after_feedback":0},"nut-knob/M5":{"passes":0,"trials":10,"not_graded":0,"passed_first":0,"passed_after_feedback":0}},"passes":2,"trials":40,"passed_first":2,"passed_after_feedback":0,"rate":0.05,"not_graded":4,"replays_replaced_by_regrade":0,"cost_usd":5.508792,"cost_per_pass":2.754396,"est_print_s_per_pass":10998.0,"est_grams_per_pass":103.825},"test_first":null,"tokens_per_trial":69608.775,"change_points":null},{"model":"qwen/qwen3.8-flash","short":"Qwen 3.8 Flash","replayed":false,"first":{"by_task":{"tripod-plate":{"passes":0,"trials":10,"not_graded":0},"bearing-wheel":{"passes":1,"trials":10,"not_graded":0},"nut-knob/M3":{"passes":0,"trials":10,"not_graded":0},"nut-knob/M5":{"passes":0,"trials":5,"not_graded":1}},"passes":1,"trials":35,"rate":0.02857142857142857,"not_graded":1,"excluded":{"count":0,"by_reason":[],"by_task":{}}},"after_feedback":{"by_task":{"tripod-plate":{"passes":0,"trials":10,"not_graded":0,"passed_first":0,"passed_after_feedback":0},"bearing-wheel":{"passes":1,"trials":10,"not_graded":0,"passed_first":1,"passed_after_feedback":0},"nut-knob/M3":{"passes":0,"trials":10,"not_graded":0,"passed_first":0,"passed_after_feedback":0},"nut-knob/M5":{"passes":0,"trials":5,"not_graded":1,"passed_first":0,"passed_after_feedback":0}},"passes":1,"trials":35,"passed_first":1,"passed_after_feedback":0,"rate":0.02857142857142857,"not_graded":1,"replays_replaced_by_regrade":0,"cost_usd":1.76528,"cost_per_pass":1.76528,"est_print_s_per_pass":24957.0,"est_grams_per_pass":234.51},"test_first":null,"tokens_per_trial":299471.17142857146,"change_points":null},{"model":"anthropic/claude-haiku-5.5","short":"Haiku 5.5","replayed":false,"first":{"by_task":{"tripod-plate":{"passes":0,"trials":10,"not_graded":0},"bearing-wheel":{"passes":0,"trials":10,"not_graded":0},"nut-knob/M3":{"passes":0,"trials":10,"not_graded":3},"nut-knob/M5":{"passes":1,"trials":10,"not_graded":0}},"passes":1,"trials":40,"rate":0.025,"not_graded":3,"excluded":{"count":0,"by_reason":[],"by_task":{}}},"after_feedback":{"by_task":{"tripod-plate":{"passes":0,"trials":10,"not_graded":0,"passed_first":0,"passed_after_feedback":0},"bearing-wheel":{"passes":0,"trials":10,"not_graded":0,"passed_first":0,"passed_after_feedback":0},"nut-knob/M3":{"passes":0,"trials":10,"not_graded":3,"passed_first":0,"passed_after_feedback":0},"nut-knob/M5":{"passes":1,"trials":10,"not_graded":0,"passed_first":1,"passed_after_feedback":0}},"passes":1,"trials":40,"passed_first":1,"passed_after_feedback":0,"rate":0.025,"not_graded":3,"replays_replaced_by_regrade":0,"cost_usd":0.215947,"cost_per_pass":0.215947,"est_print_s_per_pass":25007.0,"est_grams_per_pass":259.7},"test_first":{"by_task":{"tripod-plate":{"passes":0,"trials":5,"not_graded":0,"cant_tell":{"trials":0,"passes":0}},"bearing-wheel":{"passes":0,"trials":5,"not_graded":0,"cant_tell":{"trials":2,"passes":0}},"nut-knob/M3":{"passes":2,"trials":5,"not_graded":3,"cant_tell":{"trials":2,"passes":1}},"nut-knob/M5":{"passes":1,"trials":5,"not_graded":0,"cant_tell":{"trials":4,"passes":1}},"estoa-light":{"passes":0,"trials":0,"not_graded":0,"cant_tell":{"trials":0,"passes":0}}},"passes":3,"trials":20,"rate":0.15,"cost_per_pass":0.043563,"est_print_s_per_pass":7109.666666666667,"est_grams_per_pass":73.92666666666666,"cant_tell":{"trials":8,"passes":2},"excluded":{"count":0,"by_reason":[],"by_task":{}}},"tokens_per_trial":34237.85,"change_points":12}],"test_first_order":["google/gemini-4-argon","openai/gpt-6.1-sol","google/gemini-3.8-flash","anthropic/claude-opus-5.5","anthropic/claude-sonnet-5.5","deepseek/deepseek-v4.1-flash","anthropic/claude-haiku-5.5"],"not_tested_order":["z-ai/glm-5.3-flash","openai/gpt-6-luna","deepseek/deepseek-v4-pro-0813","qwen/qwen3.8-flash"],"pooled":{"models":7,"model_ids":["google/gemini-4-argon","openai/gpt-6.1-sol","google/gemini-3.8-flash","anthropic/claude-opus-5.5","anthropic/claude-sonnet-5.5","deepseek/deepseek-v4.1-flash","anthropic/claude-haiku-5.5"],"first":{"passes":105,"trials":281,"rate":0.3736654804270463},"after_feedback":{"passes":150,"trials":241,"rate":0.6224066390041494,"models":6,"model_ids":["google/gemini-4-argon","openai/gpt-6.1-sol","google/gemini-3.8-flash","anthropic/claude-opus-5.5","anthropic/claude-sonnet-5.5","deepseek/deepseek-v4.1-flash"],"first":{"passes":104,"trials":241,"rate":0.4315352697095436},"not_sent_back":["Haiku 5.5"]},"test_first":{"passes":116,"trials":219,"rate":0.5296803652968036},"first_rate":0.3736654804270463,"test_first_rate":0.5296803652968036},"matched_trial_numbers":{"note":"First print against test-first over the same model, task and trial numbers only: the trials both conditions have. Trial n in one condition has nothing to do with trial n in the other; this only evens out how many trials each model and task contributes (test-first lost some to out-of-credit errors). Not a scorecard.py statistic.","first":{"passes":86,"trials":219,"rate":0.3926940639269406},"test_first":{"passes":116,"trials":219,"rate":0.5296803652968036},"per_model":{"anthropic/claude-haiku-5.5":{"first":{"passes":1,"trials":20,"rate":0.05},"test_first":{"passes":3,"trials":20,"rate":0.15}},"anthropic/claude-opus-5.5":{"first":{"passes":16,"trials":40,"rate":0.4},"test_first":{"passes":21,"trials":40,"rate":0.525}},"anthropic/claude-sonnet-5.5":{"first":{"passes":14,"trials":40,"rate":0.35},"test_first":{"passes":20,"trials":40,"rate":0.5}},"deepseek/deepseek-v4.1-flash":{"first":{"passes":10,"trials":26,"rate":0.38461538461538464},"test_first":{"passes":10,"trials":26,"rate":0.38461538461538464}},"google/gemini-3.8-flash":{"first":{"passes":12,"trials":25,"rate":0.48},"test_first":{"passes":14,"trials":25,"rate":0.56}},"google/gemini-4-argon":{"first":{"passes":13,"trials":28,"rate":0.4642857142857143},"test_first":{"passes":23,"trials":28,"rate":0.8214285714285714}},"openai/gpt-6.1-sol":{"first":{"passes":20,"trials":40,"rate":0.5},"test_first":{"passes":25,"trials":40,"rate":0.625}}}},"cant_tell":{"trials":219,"told":155,"passes":116,"passes_after_cant_tell":78},"estoa_line":"**The Estoa light box**, the harder task, isn't in the four-task total: 6 models, 30 trials. 5 of 30 designs pass on the first print. In the Reprint after feedback condition, after the person says how the box came out, 12 of the 26 designs replayed (every one that had failed when the replays ran) pass. With test prints first, 4 of 18 pass (its row in the table). We ran out of time to make the test-print loop work on this task ([the Estoa section](#estoa-light-box-harder-task))."},"definitions":{"pass":"Every counted check passes on Dan's recorded test prints.","fail":"At least one counted check fails.","not graded":"No recorded print decides at least one check, and none fails. Never counted as a pass.","not reached":"A check that couldn't be read because an earlier failure blocked it (for example the part didn't build). Shown apart from not graded; the trial's verdict and every count are unchanged.","declined":"The model handed in no final part on purpose. Neither a pass nor a fail; never counted as a pass.","no design":"The model never handed in a design.","error":"The trial stopped for an infrastructure reason (for example out of API credit), so it says nothing about the model and is left out of the counts.","not counted":"Shown for completeness but left out of the scorecard, with the reason beside it.","cost per passing design":"What the API provider charged for all of the model's trials in that condition, test prints and second prints included, divided by the number of passes.","print time and filament":"Estimates from the slicer (Bambu Studio) for everything the model handed in, final parts and test prints, as if each were printed once. Nothing was actually printed.","Dan":"The eval's author. He printed the test pieces whose recorded results the graders and the simulated person use.","simulated person":"The person the model talks to is a program, not a human: it answers measurements from a fixed list and says how a print came out only from Dan's recorded test prints, or the bridge check's prediction where none was printed.","test print":"A small piece the model asks for to settle a size before its final design."},"tasks":{"bearing-wheel":{"label":"Closet door wheel","short":"Wheel","scored":true,"harder":false},"estoa-light":{"label":"Estoa light box (harder)","short":"Estoa box","scored":false,"harder":true},"m5-spacer":{"label":"M5 spacer (control)","short":"Spacer","scored":false,"harder":false},"nut-knob/M3":{"label":"Hand knob, M3","short":"Knob M3","scored":true,"harder":false},"nut-knob/M5":{"label":"Hand knob, M5","short":"Knob M5","scored":true,"harder":false},"tripod-plate":{"label":"Tripod plate","short":"Tripod","scored":true,"harder":false}},"models":{"anthropic/claude-haiku-5.5":{"short":"Haiku 5.5","listed":true,"scorecard_rank":11,"frontier":false},"anthropic/claude-opus-5.5":{"short":"Opus 5.5","listed":true,"scorecard_rank":2,"frontier":true},"anthropic/claude-sonnet-5.5":{"short":"Sonnet 5.5","listed":true,"scorecard_rank":6,"frontier":true},"deepseek/deepseek-v4-pro-0813":{"short":"DeepSeek V4 Pro","listed":true,"scorecard_rank":9,"frontier":true},"deepseek/deepseek-v4.1-flash":{"short":"DeepSeek V4.1 Flash","listed":true,"scorecard_rank":4,"frontier":false},"google/gemini-3.8-flash":{"short":"Gemini 3.8 Flash","listed":true,"scorecard_rank":3,"frontier":false},"google/gemini-4-argon":{"short":"Gemini 4 Argon","listed":true,"scorecard_rank":1,"frontier":true},"google/gemini-4-carbon":{"short":"Gemini 4 Carbon","listed":false,"scorecard_rank":null,"frontier":false},"openai/gpt-6-luna":{"short":"GPT-6 Luna","listed":true,"scorecard_rank":8,"frontier":false},"openai/gpt-6.1-sol":{"short":"GPT-6.1 Sol","listed":true,"scorecard_rank":5,"frontier":true},"qwen/qwen3.8-flash":{"short":"Qwen 3.8 Flash","listed":true,"scorecard_rank":10,"frontier":false},"x-ai/grok-4.7":{"short":"Grok 4.7","listed":false,"scorecard_rank":null,"frontier":true},"z-ai/glm-5.3-flash":{"short":"GLM 5.3 Flash","listed":true,"scorecard_rank":7,"frontier":false}},"notes":{"printing":"No complete model design was printed (round 3 printed six roof sections cut from models' Estoa designs). Each design is graded by code against Dan's recorded test prints; the simulated person's answers come from those records, or the bridge check's prediction where none was printed.","not_graded":"Not graded means no recorded print decides a check. It is never a pass.","test_print":"A test print is a small piece the model asks for to settle a size before its final design."},"groups":[{"key":"first","label":"First print","description":"The model can ask the person for measurements, then hands in one design. It can't ask for test prints.","category":"headline","kind":"headline","runs":["pilot-tripod","pilot-wheel","pilot-nut-knob","pilot-nut-knob-2","pilot-estoa"],"condition_key":"can_ask","trials":466},{"key":"second","label":"Second prints","description":"Only the first prints that didn't pass, each sent back once: the same conversation, continued with one more message in which the simulated person says how everything in the first design came out, using only Dan's recorded prints. The model then hands in a revised design, graded with its test prints left out.","category":"headline","kind":"headline","runs":["loop-all","loop-estoa"],"condition_key":null,"trials":212},{"key":"test_first","label":"Test prints first","description":"The model can ask the person to print small test pieces before its final design, and hears how each came out, using only Dan's recorded prints.","category":"headline","kind":"headline","runs":["test-first"],"condition_key":"can_test","trials":278}]}