forked from getagentseal/codeburn
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmodel-efficiency.ts
More file actions
60 lines (51 loc) · 2.04 KB
/
Copy pathmodel-efficiency.ts
File metadata and controls
60 lines (51 loc) · 2.04 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
import { getShortModelName } from './models.js'
import type { ProjectSummary } from './types.js'
export type ModelEfficiency = {
model: string
editTurns: number
oneShotTurns: number
retries: number
editCostUSD: number
oneShotRate: number | null
retriesPerEdit: number | null
costPerEditUSD: number | null
}
type MutableModelEfficiency = Omit<ModelEfficiency, 'oneShotRate' | 'retriesPerEdit' | 'costPerEditUSD'>
function rate(num: number, den: number): number | null {
if (den === 0) return null
return Math.round((num / den) * 1000) / 10
}
export function aggregateModelEfficiency(projects: ProjectSummary[]): Map<string, ModelEfficiency> {
const byModel = new Map<string, MutableModelEfficiency>()
function ensure(model: string): MutableModelEfficiency {
let stats = byModel.get(model)
if (!stats) {
stats = { model, editTurns: 0, oneShotTurns: 0, retries: 0, editCostUSD: 0 }
byModel.set(model, stats)
}
return stats
}
for (const project of projects) {
for (const session of project.sessions) {
for (const turn of session.turns) {
if (!turn.hasEdits || turn.assistantCalls.length === 0) continue
const primaryCall = turn.assistantCalls.find(c => getShortModelName(c.model) !== '<synthetic>')
if (!primaryCall) continue
const primaryModel = getShortModelName(primaryCall.model)
const stats = ensure(primaryModel)
stats.editTurns++
if (turn.retries === 0) stats.oneShotTurns++
stats.retries += turn.retries
stats.editCostUSD += turn.assistantCalls.reduce((sum, call) => {
return getShortModelName(call.model) === '<synthetic>' ? sum : sum + call.costUSD
}, 0)
}
}
}
return new Map([...byModel.entries()].map(([model, stats]) => [model, {
...stats,
oneShotRate: rate(stats.oneShotTurns, stats.editTurns),
retriesPerEdit: stats.editTurns > 0 ? Math.round((stats.retries / stats.editTurns) * 10) / 10 : null,
costPerEditUSD: stats.editTurns > 0 ? stats.editCostUSD / stats.editTurns : null,
}]))
}