{"benchmarks":[{"key":"deepswe","title":"DeepSWE v1.1","note":"mini-swe-agent, avg@3","format":"num2","results":{"flash":{"1":48.67,"2":53.1,"3":56.78,"4":54.03,"5":57.23,"6":54.57,"8":57.08,"10":60.18,"11":54.13,"12":60.77,"13":57.52,"14":59.59,"15":60.77,"16":63.86,"17":58.11,"18":64.01,"19":61.65,"20":63.72,"21":59.59,"22":64.01,"23":61.36,"24":65.78,"25":64.9,"26":66.08,"27":66.08,"28":67.86,"29":65.78,"30":65.68},"pro":{"1":58.41,"2":56.25,"3":58.41,"4":60.47,"5":59.59,"6":58.55,"7":57.44,"8":62.24,"10":63.72,"11":65.78,"12":65.97,"13":63.27,"14":64.6,"15":63.42,"16":65.18,"17":66.37,"18":67.46,"19":67.95,"20":68.05,"21":68.15,"22":70.92,"23":70.36,"24":69.35,"25":68.56,"26":72.57,"27":69.54,"28":68.25,"29":71.09,"30":72.57}}},{"key":"inhouse-coding","title":"In-house Coding Bench","note":"avg@3","format":"num2","results":{"flash":{"1":53.83,"2":54.69,"3":55.02,"4":55.34,"5":56.41,"6":56.5,"8":57.47,"10":58.23,"11":57.45,"12":59.22,"13":58.81,"14":59.19,"15":58.96,"16":60.34,"17":59.96,"18":60.92,"19":61.09,"20":61.28,"21":61.33,"22":61.02,"23":61.53,"24":62.28,"25":61.93,"26":62.19,"27":62.49,"28":62.91,"29":62.47,"30":62.87},"pro":{"1":57.54,"2":57.2,"3":57.26,"4":58.9,"5":58.74,"6":59.13,"7":59.23,"8":59.25,"9":60.73,"10":61.38,"11":61.58,"12":61.64,"13":62.54,"14":62.54,"15":62.26,"16":61.68,"18":63.67,"20":64.62,"21":64.45,"22":64.44,"23":64.73,"24":64.41,"25":65.14,"26":64.39,"27":65.1,"28":64.84,"29":65.43}}},{"key":"automation","title":"AutomationBench v1.0.6","note":"avg@3","format":"num2","results":{"flash":{"1":44.8,"2":44.7,"3":44.7,"4":45.4,"5":45.8,"6":45.6,"7":46.6,"8":47.4,"9":47.6,"10":47.6,"11":48.6,"12":48.6,"13":48.2,"14":48.5,"15":49.9,"16":49.4,"17":50.1,"18":50.4,"19":50.8,"20":50.2,"21":49.9,"22":50.8,"23":51.2,"24":52.3,"25":51.6,"26":52.4,"27":52.2,"28":52.3,"29":52.8,"30":52.7},"pro":{"1":45.2,"2":45.1,"3":45.9,"4":45.8,"5":46.2,"6":47.1,"7":46.3,"8":47.7,"9":47.2,"10":47.8,"11":50.1,"12":48.9,"13":49.1,"14":49.8,"15":50.7,"16":49.8,"17":51.4,"18":50.7,"19":50.5,"20":50.4,"21":51.4,"22":51.0,"23":52.1,"24":51.9,"26":51.3,"29":53.1}}}]}