{"rows":[{"id":"baseten:zai-org/GLM-5.3-Flash","model":"GLM 5.3 Flash","score":91.35638060883751,"rubric":{"version":"workhorse-provisional-v2","costWeight":5,"turnsWeight":1,"errorExponent":2,"resourceBasis":"mean-per-attempt","referenceTurns":6,"referenceCostUsd":0.1,"costNormalization":"reference-over-reference-plus-mean","turnsNormalization":"capped-reference-over-mean"},"accuracy":1,"attempts":1,"failures":0,"provider":"Baseten","costBasis":["catalog-rate-estimate"],"errorRate":0,"meanTurns":8,"successes":1,"efficiency":0.9135638060883751,"totalTurns":8,"meanCostUsd":"0.005677350000","errorPenalty":1,"totalCostUsd":"0.005677350000","configuration":{"image":"europe-west3-docker.pkg.dev/kylace-state-of-inference/inference/worker@sha256:63f83ebe7ad4dc4c1997a420ec59facfd739550463d713cd781cd85d40efb480","modelId":"zai-org/GLM-5.3-Flash","revision":null,"settings":{"max_tokens":8192,"temperature":0},"fxVersion":"0.0.10","cacheRegime":"Provider-managed, fresh conversation, no controlled cache priming","quantization":"fp8","adapterVersion":"baseten-buffered-v1"},"costEfficiency":0.94627656730605,"turnEfficiency":0.75,"turnsPerSuccess":8,"accuracyWilson95":[0.20654931437723745,1],"costPerSuccessUsd":"0.005677350000","errorRateWilson95":[0,0.7934506856227626]},{"id":"baseten:deepseek-ai/DeepSeek-V4.1-Flash","model":"DeepSeek V4.1 Flash","score":87.47255854412167,"rubric":{"version":"workhorse-provisional-v2","costWeight":5,"turnsWeight":1,"errorExponent":2,"resourceBasis":"mean-per-attempt","referenceTurns":6,"referenceCostUsd":0.1,"costNormalization":"reference-over-reference-plus-mean","turnsNormalization":"capped-reference-over-mean"},"accuracy":1,"attempts":1,"failures":0,"provider":"Baseten","costBasis":["catalog-rate-estimate"],"errorRate":0,"meanTurns":8,"successes":1,"efficiency":0.8747255854412167,"totalTurns":8,"meanCostUsd":"0.011151780000","errorPenalty":1,"totalCostUsd":"0.011151780000","configuration":{"image":"europe-west3-docker.pkg.dev/kylace-state-of-inference/inference/worker@sha256:63f83ebe7ad4dc4c1997a420ec59facfd739550463d713cd781cd85d40efb480","modelId":"deepseek-ai/DeepSeek-V4.1-Flash","revision":null,"settings":{"max_tokens":8192,"temperature":0},"fxVersion":"0.0.10","cacheRegime":"Provider-managed, fresh conversation, no controlled cache priming","quantization":"fp8","adapterVersion":"baseten-buffered-v1"},"costEfficiency":0.8996707025294601,"turnEfficiency":0.75,"turnsPerSuccess":8,"accuracyWilson95":[0.20654931437723745,1],"costPerSuccessUsd":"0.011151780000","errorRateWilson95":[0,0.7934506856227626]},{"id":"baseten:zai-org/GLM-5.3","model":"GLM 5.3","score":78.76237078705172,"rubric":{"version":"workhorse-provisional-v2","costWeight":5,"turnsWeight":1,"errorExponent":2,"resourceBasis":"mean-per-attempt","referenceTurns":6,"referenceCostUsd":0.1,"costNormalization":"reference-over-reference-plus-mean","turnsNormalization":"capped-reference-over-mean"},"accuracy":1,"attempts":1,"failures":0,"provider":"Baseten","costBasis":["catalog-rate-estimate"],"errorRate":0,"meanTurns":8,"successes":1,"efficiency":0.7876237078705172,"totalTurns":8,"meanCostUsd":"0.025762680000","errorPenalty":1,"totalCostUsd":"0.025762680000","configuration":{"image":"europe-west3-docker.pkg.dev/kylace-state-of-inference/inference/worker@sha256:63f83ebe7ad4dc4c1997a420ec59facfd739550463d713cd781cd85d40efb480","modelId":"zai-org/GLM-5.3","revision":null,"settings":{"max_tokens":8192,"temperature":0},"fxVersion":"0.0.10","cacheRegime":"Provider-managed, fresh conversation, no controlled cache priming","quantization":"fp4","adapterVersion":"baseten-buffered-v1"},"costEfficiency":0.7951484494446206,"turnEfficiency":0.75,"turnsPerSuccess":8,"accuracyWilson95":[0.20654931437723745,1],"costPerSuccessUsd":"0.025762680000","errorRateWilson95":[0,0.7934506856227626]},{"id":"baseten:deepseek-ai/DeepSeek-V4-Pro-0813","model":"DeepSeek V4 Pro 0813","score":63.59878744621342,"rubric":{"version":"workhorse-provisional-v2","costWeight":5,"turnsWeight":1,"errorExponent":2,"resourceBasis":"mean-per-attempt","referenceTurns":6,"referenceCostUsd":0.1,"costNormalization":"reference-over-reference-plus-mean","turnsNormalization":"capped-reference-over-mean"},"accuracy":1,"attempts":1,"failures":0,"provider":"Baseten","costBasis":["catalog-rate-estimate"],"errorRate":0,"meanTurns":8,"successes":1,"efficiency":0.6359878744621342,"totalTurns":8,"meanCostUsd":"0.063082800000","errorPenalty":1,"totalCostUsd":"0.063082800000","configuration":{"image":"europe-west3-docker.pkg.dev/kylace-state-of-inference/inference/worker@sha256:63f83ebe7ad4dc4c1997a420ec59facfd739550463d713cd781cd85d40efb480","modelId":"deepseek-ai/DeepSeek-V4-Pro-0813","revision":null,"settings":{"max_tokens":8192,"temperature":0},"fxVersion":"0.0.10","cacheRegime":"Provider-managed, fresh conversation, no controlled cache priming","quantization":"fp4","adapterVersion":"baseten-buffered-v1"},"costEfficiency":0.613185449354561,"turnEfficiency":0.75,"turnsPerSuccess":8,"accuracyWilson95":[0.20654931437723745,1],"costPerSuccessUsd":"0.063082800000","errorRateWilson95":[0,0.7934506856227626]},{"id":"baseten:moonshotai/Kimi-K3","model":"Kimi K3","score":50.984072366065426,"rubric":{"version":"workhorse-provisional-v2","costWeight":5,"turnsWeight":1,"errorExponent":2,"resourceBasis":"mean-per-attempt","referenceTurns":6,"referenceCostUsd":0.1,"costNormalization":"reference-over-reference-plus-mean","turnsNormalization":"capped-reference-over-mean"},"accuracy":1,"attempts":1,"failures":0,"provider":"Baseten","costBasis":["catalog-rate-estimate"],"errorRate":0,"meanTurns":8,"successes":1,"efficiency":0.5098407236606542,"totalTurns":8,"meanCostUsd":"0.116539800000","errorPenalty":1,"totalCostUsd":"0.116539800000","configuration":{"image":"europe-west3-docker.pkg.dev/kylace-state-of-inference/inference/worker@sha256:63f83ebe7ad4dc4c1997a420ec59facfd739550463d713cd781cd85d40efb480","modelId":"moonshotai/Kimi-K3","revision":null,"settings":{"max_tokens":8192,"temperature":0},"fxVersion":"0.0.10","cacheRegime":"Provider-managed, fresh conversation, no controlled cache priming","quantization":"fp8","adapterVersion":"baseten-buffered-v1"},"costEfficiency":0.46180886839278507,"turnEfficiency":0.75,"turnsPerSuccess":8,"accuracyWilson95":[0.20654931437723745,1],"costPerSuccessUsd":"0.116539800000","errorRateWilson95":[0,0.7934506856227626]}],"seeds":[102],"rubric":{"version":"workhorse-provisional-v2","costWeight":5,"turnsWeight":1,"errorExponent":2,"resourceBasis":"mean-per-attempt","referenceTurns":6,"referenceCostUsd":0.1,"costNormalization":"reference-over-reference-plus-mean","turnsNormalization":"capped-reference-over-mean"},"status":"pilot","channel":"benchmark","runtime":"linux-amd64","evidence":[{"id":"fabbf8b3-8c8b-4498-8237-44980108e025","url":"/api/state-of-inference/artifacts/fabbf8b3-8c8b-4498-8237-44980108e025","name":"report.json","seed":102,"bytes":"6540","sha256":"440e9c2b6bbf28bd68abc97f9c625e499430494e759d42e69339e24351ee3168","modelId":"deepseek-ai/DeepSeek-V4.1-Flash","episodeId":"78b9e3e8-0883-4abb-876b-26ecc2c3a02b","repetition":1},{"id":"fe2b57f4-4b27-43b7-ae38-4ae91cfb3236","url":"/api/state-of-inference/artifacts/fe2b57f4-4b27-43b7-ae38-4ae91cfb3236","name":"report.pdf","seed":102,"bytes":"4100","sha256":"8dd9efc46bd39ba6b5f1574976e419a6beabfd4b784e19a9aec302ded676c0a0","modelId":"deepseek-ai/DeepSeek-V4.1-Flash","episodeId":"78b9e3e8-0883-4abb-876b-26ecc2c3a02b","repetition":1},{"id":"5153efce-8951-484c-987e-08b1ebacb7ee","url":"/api/state-of-inference/artifacts/5153efce-8951-484c-987e-08b1ebacb7ee","name":"sources/expenses.json","seed":102,"bytes":"875","sha256":"62e23608aa09a5ed5d2e913ba4c29b82d1459675273afa4455e812aeb948d438","modelId":"deepseek-ai/DeepSeek-V4.1-Flash","episodeId":"78b9e3e8-0883-4abb-876b-26ecc2c3a02b","repetition":1},{"id":"d16f461f-c869-4ab9-809a-b7c3f773d9cc","url":"/api/state-of-inference/artifacts/d16f461f-c869-4ab9-809a-b7c3f773d9cc","name":"sources/invoices.json","seed":102,"bytes":"1289","sha256":"0488183137539c64eb09a6b77907dd807a4ea44b3c863a415df60f564dee970f","modelId":"deepseek-ai/DeepSeek-V4.1-Flash","episodeId":"78b9e3e8-0883-4abb-876b-26ecc2c3a02b","repetition":1},{"id":"7d527cc1-140b-436e-9b12-feceb557989b","url":"/api/state-of-inference/artifacts/7d527cc1-140b-436e-9b12-feceb557989b","name":"sources/payments.json","seed":102,"bytes":"1192","sha256":"bdcc4af56413a6c17ec3138bdd4fbb7d9441878f445b42e9dbd56b80cd10eae4","modelId":"deepseek-ai/DeepSeek-V4.1-Flash","episodeId":"78b9e3e8-0883-4abb-876b-26ecc2c3a02b","repetition":1},{"id":"98d06e76-411e-4e12-a4f1-c30021c65ccb","url":"/api/state-of-inference/artifacts/98d06e76-411e-4e12-a4f1-c30021c65ccb","name":"report.json","seed":102,"bytes":"6540","sha256":"440e9c2b6bbf28bd68abc97f9c625e499430494e759d42e69339e24351ee3168","modelId":"zai-org/GLM-5.3","episodeId":"ef7ecce0-9e2b-47e1-b30d-cf39bfde83e6","repetition":1},{"id":"5d45d80f-4e42-40db-80e9-36449f200f71","url":"/api/state-of-inference/artifacts/5d45d80f-4e42-40db-80e9-36449f200f71","name":"report.pdf","seed":102,"bytes":"4100","sha256":"8dd9efc46bd39ba6b5f1574976e419a6beabfd4b784e19a9aec302ded676c0a0","modelId":"zai-org/GLM-5.3","episodeId":"ef7ecce0-9e2b-47e1-b30d-cf39bfde83e6","repetition":1},{"id":"68cc1ae8-50e8-4de0-98ca-b708b85489ec","url":"/api/state-of-inference/artifacts/68cc1ae8-50e8-4de0-98ca-b708b85489ec","name":"sources/expenses.json","seed":102,"bytes":"875","sha256":"62e23608aa09a5ed5d2e913ba4c29b82d1459675273afa4455e812aeb948d438","modelId":"zai-org/GLM-5.3","episodeId":"ef7ecce0-9e2b-47e1-b30d-cf39bfde83e6","repetition":1},{"id":"2ad8872c-357e-4921-a50b-e887826385d0","url":"/api/state-of-inference/artifacts/2ad8872c-357e-4921-a50b-e887826385d0","name":"sources/invoices.json","seed":102,"bytes":"1289","sha256":"0488183137539c64eb09a6b77907dd807a4ea44b3c863a415df60f564dee970f","modelId":"zai-org/GLM-5.3","episodeId":"ef7ecce0-9e2b-47e1-b30d-cf39bfde83e6","repetition":1},{"id":"b213b0f1-f4fc-4ac4-b61d-557bef559910","url":"/api/state-of-inference/artifacts/b213b0f1-f4fc-4ac4-b61d-557bef559910","name":"sources/payments.json","seed":102,"bytes":"1192","sha256":"bdcc4af56413a6c17ec3138bdd4fbb7d9441878f445b42e9dbd56b80cd10eae4","modelId":"zai-org/GLM-5.3","episodeId":"ef7ecce0-9e2b-47e1-b30d-cf39bfde83e6","repetition":1},{"id":"562ea348-1db8-4652-8e9f-84f9689ca056","url":"/api/state-of-inference/artifacts/562ea348-1db8-4652-8e9f-84f9689ca056","name":"report.json","seed":102,"bytes":"6540","sha256":"2cc1f2426b9c3af43bf0dc42c90cd248f48510af461902ef46b7ff33de6b16c0","modelId":"zai-org/GLM-5.3-Flash","episodeId":"9d0cb71f-6994-4c83-acae-f68423aaea2b","repetition":1},{"id":"774b953f-641a-42a5-84cb-4c7e4d410856","url":"/api/state-of-inference/artifacts/774b953f-641a-42a5-84cb-4c7e4d410856","name":"report.pdf","seed":102,"bytes":"4100","sha256":"8dd9efc46bd39ba6b5f1574976e419a6beabfd4b784e19a9aec302ded676c0a0","modelId":"zai-org/GLM-5.3-Flash","episodeId":"9d0cb71f-6994-4c83-acae-f68423aaea2b","repetition":1},{"id":"1ee7ac41-ec00-4845-ba26-a1415f741409","url":"/api/state-of-inference/artifacts/1ee7ac41-ec00-4845-ba26-a1415f741409","name":"sources/expenses.json","seed":102,"bytes":"875","sha256":"62e23608aa09a5ed5d2e913ba4c29b82d1459675273afa4455e812aeb948d438","modelId":"zai-org/GLM-5.3-Flash","episodeId":"9d0cb71f-6994-4c83-acae-f68423aaea2b","repetition":1},{"id":"b7bdc1ef-ff6c-428d-ae49-c4e09f1141fe","url":"/api/state-of-inference/artifacts/b7bdc1ef-ff6c-428d-ae49-c4e09f1141fe","name":"sources/invoices.json","seed":102,"bytes":"1289","sha256":"0488183137539c64eb09a6b77907dd807a4ea44b3c863a415df60f564dee970f","modelId":"zai-org/GLM-5.3-Flash","episodeId":"9d0cb71f-6994-4c83-acae-f68423aaea2b","repetition":1},{"id":"fec03041-ee3c-47cf-9f2e-36410f486925","url":"/api/state-of-inference/artifacts/fec03041-ee3c-47cf-9f2e-36410f486925","name":"sources/payments.json","seed":102,"bytes":"1192","sha256":"bdcc4af56413a6c17ec3138bdd4fbb7d9441878f445b42e9dbd56b80cd10eae4","modelId":"zai-org/GLM-5.3-Flash","episodeId":"9d0cb71f-6994-4c83-acae-f68423aaea2b","repetition":1},{"id":"bf2100c8-28ec-416a-8ce2-26ae1a595729","url":"/api/state-of-inference/artifacts/bf2100c8-28ec-416a-8ce2-26ae1a595729","name":"report.json","seed":102,"bytes":"6540","sha256":"440e9c2b6bbf28bd68abc97f9c625e499430494e759d42e69339e24351ee3168","modelId":"deepseek-ai/DeepSeek-V4-Pro-0813","episodeId":"44bb10f1-23da-4b56-b717-9b25773ceb48","repetition":1},{"id":"95de94dc-c7f0-4981-98a4-4ad2f85c2b54","url":"/api/state-of-inference/artifacts/95de94dc-c7f0-4981-98a4-4ad2f85c2b54","name":"report.pdf","seed":102,"bytes":"4100","sha256":"8dd9efc46bd39ba6b5f1574976e419a6beabfd4b784e19a9aec302ded676c0a0","modelId":"deepseek-ai/DeepSeek-V4-Pro-0813","episodeId":"44bb10f1-23da-4b56-b717-9b25773ceb48","repetition":1},{"id":"e7c89654-33fc-4223-9ac8-65fbf8aaefb4","url":"/api/state-of-inference/artifacts/e7c89654-33fc-4223-9ac8-65fbf8aaefb4","name":"sources/expenses.json","seed":102,"bytes":"875","sha256":"62e23608aa09a5ed5d2e913ba4c29b82d1459675273afa4455e812aeb948d438","modelId":"deepseek-ai/DeepSeek-V4-Pro-0813","episodeId":"44bb10f1-23da-4b56-b717-9b25773ceb48","repetition":1},{"id":"a3c38857-bba3-477d-9292-c59c1a22a592","url":"/api/state-of-inference/artifacts/a3c38857-bba3-477d-9292-c59c1a22a592","name":"sources/invoices.json","seed":102,"bytes":"1289","sha256":"0488183137539c64eb09a6b77907dd807a4ea44b3c863a415df60f564dee970f","modelId":"deepseek-ai/DeepSeek-V4-Pro-0813","episodeId":"44bb10f1-23da-4b56-b717-9b25773ceb48","repetition":1},{"id":"15e9948b-220e-4d1a-8550-1d18ccd68c0f","url":"/api/state-of-inference/artifacts/15e9948b-220e-4d1a-8550-1d18ccd68c0f","name":"sources/payments.json","seed":102,"bytes":"1192","sha256":"bdcc4af56413a6c17ec3138bdd4fbb7d9441878f445b42e9dbd56b80cd10eae4","modelId":"deepseek-ai/DeepSeek-V4-Pro-0813","episodeId":"44bb10f1-23da-4b56-b717-9b25773ceb48","repetition":1},{"id":"5c2ec3eb-1e10-44d6-aad6-e0bc35e26a9e","url":"/api/state-of-inference/artifacts/5c2ec3eb-1e10-44d6-aad6-e0bc35e26a9e","name":"report.json","seed":102,"bytes":"6540","sha256":"440e9c2b6bbf28bd68abc97f9c625e499430494e759d42e69339e24351ee3168","modelId":"moonshotai/Kimi-K3","episodeId":"070863dd-19a0-4a54-8bda-85c8b73de568","repetition":1},{"id":"dd8ec5fe-e79a-4cb9-993e-8f662a2c4f24","url":"/api/state-of-inference/artifacts/dd8ec5fe-e79a-4cb9-993e-8f662a2c4f24","name":"report.pdf","seed":102,"bytes":"4100","sha256":"8dd9efc46bd39ba6b5f1574976e419a6beabfd4b784e19a9aec302ded676c0a0","modelId":"moonshotai/Kimi-K3","episodeId":"070863dd-19a0-4a54-8bda-85c8b73de568","repetition":1},{"id":"bb63bc8e-0881-4ac3-bb65-bd03b805ce8a","url":"/api/state-of-inference/artifacts/bb63bc8e-0881-4ac3-bb65-bd03b805ce8a","name":"sources/expenses.json","seed":102,"bytes":"875","sha256":"62e23608aa09a5ed5d2e913ba4c29b82d1459675273afa4455e812aeb948d438","modelId":"moonshotai/Kimi-K3","episodeId":"070863dd-19a0-4a54-8bda-85c8b73de568","repetition":1},{"id":"390449ef-362a-458d-b565-8451f4c854ec","url":"/api/state-of-inference/artifacts/390449ef-362a-458d-b565-8451f4c854ec","name":"sources/invoices.json","seed":102,"bytes":"1289","sha256":"0488183137539c64eb09a6b77907dd807a4ea44b3c863a415df60f564dee970f","modelId":"moonshotai/Kimi-K3","episodeId":"070863dd-19a0-4a54-8bda-85c8b73de568","repetition":1},{"id":"fc793f2f-9a92-4457-b1b8-c1c3f1df54e7","url":"/api/state-of-inference/artifacts/fc793f2f-9a92-4457-b1b8-c1c3f1df54e7","name":"sources/payments.json","seed":102,"bytes":"1192","sha256":"bdcc4af56413a6c17ec3138bdd4fbb7d9441878f445b42e9dbd56b80cd10eae4","modelId":"moonshotai/Kimi-K3","episodeId":"070863dd-19a0-4a54-8bda-85c8b73de568","repetition":1}],"scoredAt":"2026-09-14T15:10:12.537Z","seriesId":"25588a9e00ac50fc892d8875dbe11ddbff7e4178356fac07e824e5a5d94a5634","measuredAt":"2026-09-14T15:07:26.876Z","repetitions":1,"taskVersion":"financial-close-v1","sourceSha256":"601c04a2a79139731b77cefa9b4732f00afc6aa4fc1831931337bbefc8f8ec39","providerCount":1,"schemaVersion":3}