Skip to content

Commit 0a3beff

Browse files
github-actions[bot]github-actions[bot]
committed
Sync public snapshot from freebuff-private
Source: CodebuffAI/freebuff-private@09acfecc71fc59bba22ac69d454f5dcd6428e9c9
1 parent f56c92b commit 0a3beff

11 files changed

Lines changed: 557 additions & 6 deletions

‎cli/src/utils/__tests__/sponsored-run.test.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -572,6 +572,9 @@ describe('the turn', () => {
572572
await f.service.settleTurn({ errorText: 'model refused', aborted: false })
573573
await settle()
574574
expect(f.reported.map((r) => r.state)).toEqual(['failed'])
575+
expect(f.reported[0]).toMatchObject({
576+
runEvidence: { version: 1, ending: 'error' },
577+
})
575578
expect(f.inflight()).toEqual([])
576579
})
577580

‎cli/src/utils/sponsored-proposal-api.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import type { SponsoredRunEvidence } from '@codebuff/common/ads/sponsored-run-evidence'
12
import type { SponsoredProposalStep } from '@codebuff/common/ads/sponsored-proposal-view'
23
import {
34
AGENTIC_OFFER_PATH,
@@ -141,6 +142,7 @@ export type SponsoredStateUpdate = {
141142
* The same failure, said plainly, for whoever has to fix it. Stored beside
142143
* `failureReason` and never rendered.
143144
*/
145+
runEvidence?: SponsoredRunEvidence
144146
diagnosticReason?: string
145147
}
146148

‎cli/src/utils/sponsored-run.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,8 @@
6363
* died before writing one is found by the next launch through its in-flight
6464
* record (`sponsored-run-inflight.ts`) and reported `failed` from there.
6565
*/
66+
67+
import { sponsoredRunEvidence } from '@codebuff/common/ads/sponsored-run-evidence'
6668
import {
6769
evaluateSponsoredInstallCommand,
6870
sponsoredDeclaredPackagesForProcedureSha256,
@@ -464,6 +466,7 @@ function terminalReportPayload(update: SponsoredStateUpdate): string {
464466
prUrl: update.prUrl,
465467
failureReason: update.failureReason,
466468
diagnosticReason: update.diagnosticReason,
469+
runEvidence: update.runEvidence,
467470
head: update.head,
468471
outcomes: update.outcomes,
469472
})
@@ -977,7 +980,8 @@ export class SponsoredRun {
977980
// first. The Accept is already settled upstream, so a spec that fails
978981
// is reported as a failed run (below) rather than run or dropped.
979982
const sentSpec =
980-
accepted.accept.runSpec !== undefined && accepted.accept.runSpec !== null
983+
accepted.accept.runSpec !== undefined &&
984+
accepted.accept.runSpec !== null
981985
const specCheck = sentSpec
982986
? checkSponsoredRunSpec(accepted.accept.runSpec, {
983987
procedure: accepted.accept.procedure,
@@ -1561,7 +1565,10 @@ export class SponsoredRun {
15611565
if (steps && !steps.delivered && verdict === 'delivered') {
15621566
verdict = 'partial_edits'
15631567
}
1564-
const stepReport = steps ? { steps: sponsoredStepCardStates(steps) } : {}
1568+
const stepReport = {
1569+
runEvidence: sponsoredRunEvidence(ending, steps),
1570+
...(steps ? { steps: sponsoredStepCardStates(steps) } : {}),
1571+
}
15651572
const stepDiagnostic = steps
15661573
? `; ${sponsoredStepDiagnostic(steps)}; skill source ${active.skillSource}`
15671574
: ''

‎common/src/ads/sponsored-in-place.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ export function sponsoredPartialEditsDiagnostic(
136136
how: string,
137137
fileCount: number,
138138
): string {
139-
return `${how}; the turn ended early, so its partial changes (${fileCount} file${
139+
return `${how}; ${how.startsWith('turn completed') ? 'verification failed' : 'the turn ended early'}, so its partial changes (${fileCount} file${
140140
fileCount === 1 ? '' : 's'
141141
}) were left in the workspace and not reported as delivered`
142142
}

‎common/src/ads/sponsored-run-evidence.ts‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
import type { SponsoredStepDriveResult } from './sponsored-step-driver'
2+
import type { SponsoredTurnEnding } from './sponsored-in-place'
3+
4+
/** Closed telemetry only: no paths, labels, command output, or credential values. */
5+
export const SPONSORED_BLOCKER_CATEGORIES = [
6+
'scope_check',
7+
'needs_api_key',
8+
'needs_login',
9+
'command_failed',
10+
'verification_failed',
11+
] as const
12+
export type SponsoredBlockerCategory =
13+
(typeof SPONSORED_BLOCKER_CATEGORIES)[number]
14+
export const SPONSORED_RUN_ENDINGS = [
15+
'completed',
16+
'stopped',
17+
'closed',
18+
'interrupted',
19+
'error',
20+
'grant_expired',
21+
'runner_restarted',
22+
] as const
23+
export type SponsoredRunEvidence = {
24+
version: 1
25+
ending: (typeof SPONSORED_RUN_ENDINGS)[number]
26+
checks: Array<{
27+
step: number
28+
checkId: string
29+
category: SponsoredBlockerCategory
30+
}>
31+
/** At least one evaluator result survived; absent on ending-only reports. */
32+
checksRecorded?: true
33+
truncated?: true
34+
}
35+
export const SPONSORED_EVIDENCE_CHECK_LIMIT = 20
36+
37+
/** Also used by Convex: keep this module free of runtime/zod dependencies. */
38+
export function parseSponsoredRunEvidence(
39+
value: unknown,
40+
): SponsoredRunEvidence | undefined {
41+
if (!value || typeof value !== 'object' || Array.isArray(value))
42+
return undefined
43+
const v = value as Record<string, unknown>
44+
if (
45+
Object.keys(v).some(
46+
(k) =>
47+
![
48+
'version',
49+
'ending',
50+
'checks',
51+
'checksRecorded',
52+
'truncated',
53+
].includes(k),
54+
) ||
55+
v.version !== 1 ||
56+
!(SPONSORED_RUN_ENDINGS as readonly unknown[]).includes(v.ending) ||
57+
(v.checksRecorded !== undefined && v.checksRecorded !== true) ||
58+
(v.truncated !== undefined && v.truncated !== true) ||
59+
!Array.isArray(v.checks) ||
60+
v.checks.length > SPONSORED_EVIDENCE_CHECK_LIMIT
61+
)
62+
return undefined
63+
const checks: SponsoredRunEvidence['checks'] = []
64+
for (const raw of v.checks) {
65+
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return undefined
66+
const c = raw as Record<string, unknown>
67+
if (
68+
Object.keys(c).some(
69+
(k) => !['step', 'checkId', 'category'].includes(k),
70+
) ||
71+
!Number.isInteger(c.step) ||
72+
(c.step as number) < 0 ||
73+
(c.step as number) > 100 ||
74+
typeof c.checkId !== 'string' ||
75+
!/^[a-zA-Z0-9_-]{1,64}$/.test(c.checkId) ||
76+
!(SPONSORED_BLOCKER_CATEGORIES as readonly unknown[]).includes(c.category)
77+
)
78+
return undefined
79+
checks.push({
80+
step: c.step as number,
81+
checkId: c.checkId,
82+
category: c.category as SponsoredBlockerCategory,
83+
})
84+
}
85+
return {
86+
version: 1,
87+
ending: v.ending as SponsoredRunEvidence['ending'],
88+
checks,
89+
...(v.checksRecorded ? { checksRecorded: true } : {}),
90+
...(v.truncated ? { truncated: true } : {}),
91+
}
92+
}
93+
94+
/** Recover structured evidence on idempotent retries without a new proposal column. */
95+
export function sponsoredEvidenceFromReport(
96+
payload: string | null | undefined,
97+
): SponsoredRunEvidence | undefined {
98+
try {
99+
return payload
100+
? parseSponsoredRunEvidence(JSON.parse(payload).runEvidence)
101+
: undefined
102+
} catch {
103+
return undefined
104+
}
105+
}
106+
107+
export function sponsoredRunEvidence(
108+
ending: SponsoredTurnEnding,
109+
steps?: SponsoredStepDriveResult | null,
110+
errorText?: string | null,
111+
): SponsoredRunEvidence {
112+
const checks: SponsoredRunEvidence['checks'] = []
113+
for (const step of [
114+
{ number: 0, checks: steps?.preflight ?? [] },
115+
...(steps?.steps ?? []),
116+
]) {
117+
for (const check of step.checks) {
118+
if (check.status !== 'fail') continue
119+
const category: SponsoredBlockerCategory =
120+
check.kind === 'changes_within'
121+
? 'scope_check'
122+
: check.kind === 'credential_provided'
123+
? 'needs_api_key'
124+
: check.kind === 'user_confirmed'
125+
? 'needs_login'
126+
: check.kind === 'command_succeeds' ||
127+
check.kind === 'script_passes'
128+
? 'command_failed'
129+
: 'verification_failed'
130+
// Skill ids are bounded telemetry identifiers, never arbitrary output.
131+
checks.push({
132+
step: step.number,
133+
checkId: /^[a-zA-Z0-9_-]{1,64}$/.test(check.id) ? check.id : 'unknown',
134+
category,
135+
})
136+
}
137+
}
138+
return {
139+
version: 1,
140+
ending: /cloud runner restarted/i.test(errorText ?? '')
141+
? 'runner_restarted'
142+
: ending,
143+
...(steps?.preflight?.length ||
144+
steps?.steps.some((step) => step.checks.length)
145+
? { checksRecorded: true as const }
146+
: {}),
147+
checks: checks.slice(0, SPONSORED_EVIDENCE_CHECK_LIMIT),
148+
...(checks.length > SPONSORED_EVIDENCE_CHECK_LIMIT
149+
? { truncated: true }
150+
: {}),
151+
}
152+
}

0 commit comments

Comments
 (0)