{"generatedAt":"2026-10-07T15:06:25.251Z","baseUrl":"https://api.kimiseek.app/v1","models":["deepseek-v4.1-flash","qwen3.8-max","kimi-k3","glm-5.3-flash"],"results":[{"model":"deepseek-v4.1-flash","caseId":"support-triage","title":"Customer-support triage","ok":true,"status":200,"timeToFirstTokenMs":null,"totalLatencyMs":23621,"inputTokens":118,"outputTokens":140,"costCny":"0.000475","requestId":"e69352c6-4434-4e1e-8031-e990f0dcf2ea","output":"{\"category\":\"billing\",\"priority\":\"high\",\"needs_human\":true,\"reply_language\":\"zh-CN\",\"reason\":\"用户为 API 账户充值 200 元，银行卡已扣款但余额仍为 0，属于支付到账异常/资金类问题，需人工核查支付流水与订单状态；用户今日上线，时效紧急，故优先级为 high。\"}","passed":true,"checks":["category=billing","priority=high","needs_human=true","reply_language=zh-CN"]},{"model":"deepseek-v4.1-flash","caseId":"typescript-fix","title":"TypeScript bug fix","ok":true,"status":200,"timeToFirstTokenMs":null,"totalLatencyMs":21148,"inputTokens":93,"outputTokens":128,"costCny":"0.000423","requestId":"8b0bf3cf-4331-46cd-a2f6-8d99078bb7d0","output":"type LineItem = { price: number; qty: number };\n\nexport function subtotal(items: LineItem[]): number {\n  return items.reduce((sum, item) => sum + item.price * item.qty, 0);\n}","passed":true,"checks":["uses item.qty","removes item.quantity","keeps subtotal"]},{"model":"deepseek-v4.1-flash","caseId":"sql-query","title":"Order analytics SQL","ok":true,"status":200,"timeToFirstTokenMs":null,"totalLatencyMs":8383,"inputTokens":98,"outputTokens":42,"costCny":"0.000186","requestId":"9d1fa258-af1f-4357-9438-7e58a9221bee","output":"SELECT user_id, SUM(amount) AS total_amount FROM orders WHERE status = 'paid' AND created_at >= datetime('now', '-30 days') GROUP BY user_id ORDER BY total_amount DESC;","passed":true,"checks":["sum(amount)","paid filter","grouped by user_id","sorted output","30-day filter"]},{"model":"qwen3.8-max","caseId":"support-triage","title":"Customer-support triage","ok":true,"status":200,"timeToFirstTokenMs":null,"totalLatencyMs":10042,"inputTokens":157,"outputTokens":184,"costCny":"0.004254","requestId":"2e4861ae-1bdf-4340-9873-b06a22a73ce1","output":"{\n  \"category\": \"billing\",\n  \"priority\": \"high\",\n  \"needs_human\": true,\n  \"reply_language\": \"zh-CN\",\n  \"reason\": \"用户已充值但余额未到账，且今日需上线，需尽快人工核查支付与账户余额。\"\n}","passed":true,"checks":["category=billing","priority=high","needs_human=true","reply_language=zh-CN"]},{"model":"qwen3.8-max","caseId":"typescript-fix","title":"TypeScript bug fix","ok":true,"status":200,"timeToFirstTokenMs":null,"totalLatencyMs":7703,"inputTokens":131,"outputTokens":234,"costCny":"0.004998","requestId":"6a309d6c-e290-4a62-81bb-292a1ca9b71f","output":"type LineItem = { price: number; qty: number };\n\nexport function subtotal(items: LineItem[]): number {\n  return items.reduce((sum, item) => sum + item.price * item.qty, 0);\n}","passed":true,"checks":["uses item.qty","removes item.quantity","keeps subtotal"]},{"model":"qwen3.8-max","caseId":"sql-query","title":"Order analytics SQL","ok":true,"status":200,"timeToFirstTokenMs":null,"totalLatencyMs":7417,"inputTokens":136,"outputTokens":243,"costCny":"0.005190","requestId":"6cf90fc9-132b-4922-a774-226d74192116","output":"SELECT user_id, SUM(amount) AS total_amount FROM orders WHERE status = 'paid' AND created_at >= datetime('now', '-30 days') GROUP BY user_id ORDER BY total_amount DESC;","passed":true,"checks":["sum(amount)","paid filter","grouped by user_id","sorted output","30-day filter"]},{"model":"kimi-k3","caseId":"support-triage","title":"Customer-support triage","ok":true,"status":200,"timeToFirstTokenMs":null,"totalLatencyMs":13736,"inputTokens":175,"outputTokens":306,"costCny":"0.027280","requestId":"ea2da87f-f096-4153-ae54-10d216a01074","output":"{\"category\":\"billing\",\"priority\":\"high\",\"needs_human\":true,\"reply_language\":\"zh-CN\",\"reason\":\"用户银行卡已扣款200元但API账户余额未到账，属于支付未入账问题，需人工核实充值流水；且用户当天要上线，时间紧迫，故优先级为高。\"}","passed":true,"checks":["category=billing","priority=high","needs_human=true","reply_language=zh-CN"]},{"model":"kimi-k3","caseId":"typescript-fix","title":"TypeScript bug fix","ok":true,"status":200,"timeToFirstTokenMs":null,"totalLatencyMs":32306,"inputTokens":154,"outputTokens":165,"costCny":"0.015664","requestId":"c75922f8-b1eb-4dbd-bdbd-ffe9ac0c1ea1","output":"type LineItem = { price: number; qty: number };\n\nexport function subtotal(items: LineItem[]): number {\n  return items.reduce((sum, item) => sum + item.price * item.qty, 0);\n}","passed":true,"checks":["uses item.qty","removes item.quantity","keeps subtotal"]},{"model":"kimi-k3","caseId":"sql-query","title":"Order analytics SQL","ok":true,"status":200,"timeToFirstTokenMs":null,"totalLatencyMs":39660,"inputTokens":159,"outputTokens":187,"costCny":"0.017504","requestId":"422c4b4a-9772-45ab-8323-06412552e2f9","output":"SELECT user_id, SUM(amount) AS total_amount FROM orders WHERE status = 'paid' AND created_at >= datetime('now', '-30 days') GROUP BY user_id ORDER BY total_amount DESC;","passed":true,"checks":["sum(amount)","paid filter","grouped by user_id","sorted output","30-day filter"]},{"model":"glm-5.3-flash","caseId":"support-triage","title":"Customer-support triage","ok":true,"status":200,"timeToFirstTokenMs":null,"totalLatencyMs":14512,"inputTokens":99,"outputTokens":400,"costCny":"0.000600","requestId":"b47227ec-2df2-4509-9268-134e2ce585b6","output":"{\"category\":\"billing\",\"priority\":\"high\",\"needs_human\":true,\"reply_language\":\"zh-CN\",\"reason\":\"银行已扣款但 API 账户余额未到账，属于充值对账问题，需人工核查支付流水并补账；客户当天要上线，时间敏感，优先级高。\"}","passed":true,"checks":["category=billing","priority=high","needs_human=true","reply_language=zh-CN"]},{"model":"glm-5.3-flash","caseId":"typescript-fix","title":"TypeScript bug fix","ok":true,"status":200,"timeToFirstTokenMs":null,"totalLatencyMs":24226,"inputTokens":69,"outputTokens":176,"costCny":"0.000274","requestId":"28d416bc-1b03-49b5-a8a7-e119b1e61c23","output":"type LineItem = { price: number; qty: number };\n\nexport function subtotal(items: LineItem[]): number {\n  return items.reduce((sum, item) => sum + item.price * item.qty, 0);\n}","passed":true,"checks":["uses item.qty","removes item.quantity","keeps subtotal"]},{"model":"glm-5.3-flash","caseId":"sql-query","title":"Order analytics SQL","ok":true,"status":200,"timeToFirstTokenMs":null,"totalLatencyMs":112602,"inputTokens":77,"outputTokens":191,"costCny":"0.000298","requestId":"a48957c0-e754-4648-83a8-81dc3e7322c8","output":"SELECT user_id, SUM(amount) AS total_amount FROM orders WHERE status = 'paid' AND date(created_at) >= date('now', '-30 days') GROUP BY user_id ORDER BY total_amount DESC;","passed":true,"checks":["sum(amount)","paid filter","grouped by user_id","sorted output","30-day filter"]}],"cases":[{"id":"support-triage","title":"Customer-support triage","prompt":"你是客服工单路由器。只输出 JSON，不要 Markdown，不要解释。字段必须为 category、priority、needs_human、reply_language、reason。category 只能是 billing、account、api；priority 只能是 low、medium、high；reply_language 使用 BCP-47。\n\n工单：我昨晚给 API 账户充值 200 元，银行卡已经扣款，但余额仍然是 0。我们今天要上线，能否尽快处理？"},{"id":"typescript-fix","title":"TypeScript bug fix","prompt":"只输出修复后的 TypeScript 函数，不要解释，不要 Markdown。保留函数名和类型定义。\n\ntype LineItem = { price: number; qty: number };\n\nexport function subtotal(items: LineItem[]): number {\n  return items.reduce((sum, item) => sum + item.price * item.quantity, 0);\n}"},{"id":"sql-query","title":"Order analytics SQL","prompt":"只输出一条 SQLite SQL，不要解释，不要 Markdown。统计过去 30 天内 status 为 paid 的订单，按 user_id 汇总 amount，结果列必须叫 user_id 和 total_amount，并按 total_amount 从高到低排序。\n\n表结构：orders(id INTEGER, user_id INTEGER, status TEXT, amount REAL, created_at TEXT)"}],"methodology":{"samplesPerModelAndCase":1,"publishedPromptLanguage":"zh-CN","originalRequestBodiesRetained":false,"requestParameters":null,"costBasis":"not_retained","timeToFirstToken":"not_recorded","evaluation":"historical static field/string checks, not execution or a quality ranking","limitations":["Request parameters and streaming mode were not retained in the historical artifact.","Cost fields were recorded by a runner that could use either response billing headers or an estimate; provenance was not retained.","There is one sample for each model/case pair; no success rate, percentile latency or model-wide ranking can be inferred.","Supplied Chinese prompts are the runner's published reproduction tasks. Original request bodies were not retained; English page prompts are reading translations."]}}