# T0-a: OpenRCA feasibility measurement (M1-M6). Telemetry-free, uses committed archive CSVs.
import pandas as pd, re, datetime as dt, itertools, collections, sys
FILES = ['agent-Bank.csv', 'agent-Telecom.csv',
'agent-Market-cloudbed-1.csv', 'agent-Market-cloudbed-2.csv']
# which fields each task asks for
ASK = {'task_1': ('time',), 'task_2': ('reason',), 'task_3': ('comp',),
'task_4': ('time', 'reason'), 'task_5': ('time', 'comp'),
'task_6': ('comp', 'reason'), 'task_7': ('time', 'comp', 'reason')}
df = pd.concat([pd.read_csv(f).assign(sysname=f.replace('agent-', '').replace('.csv', ''))
for f in FILES], ignore_index=True)
# ---- parse ground truth ----
def gt_field(g, key):
m = re.search(rf'^{key}:\s*(.+)$', g, re.M)
return m.group(1).strip() if m else None
df['gt_comp'] = df.groundtruth.map(lambda g: gt_field(g, 'component'))
df['gt_reason'] = df.groundtruth.map(lambda g: gt_field(g, 'reason'))
df['gt_dt'] = pd.to_datetime(df.groundtruth.map(lambda g: gt_field(g, 'datetime')))
# ---- parse question window from instruction ----
MONTHS = {m: i + 1 for i, m in enumerate(
['January','February','March','April','May','June','July',
'August','September','October','November','December'])}
def parse_window(s):
md = re.search(r'(January|February|March|April|May|June|July|August|September|October|November|December)\s+(\d{1,2}),\s*(\d{4})', s)
tm = re.search(r'(\d{1,2}:\d{2})\s*(?:to|and|-)\s*(\d{1,2}:\d{2})', s)
if not (md and tm):
return None, None
y, mo, d = int(md.group(3)), MONTHS[md.group(1)], int(md.group(2))
def mk(t):
h, mi = map(int, t.split(':'))
return dt.datetime(y, mo, d, h % 24, mi)
a, b = mk(tm.group(1)), mk(tm.group(2))
if b <= a:
b += dt.timedelta(days=1)
return a, b
w = df.instruction.map(parse_window)
df['win_start'] = [x[0] for x in w]
df['win_end'] = [x[1] for x in w]
print('window parse failures:', df.win_start.isna().sum(), '/', len(df))
# ================= M1 =================
print('\n===== M1: task structure =====')
n = len(df)
for t in sorted(ASK):
k = (df.task_index == t).sum()
print(f' {t}: asks {"+".join(ASK[t]):22s} n={k:3d} {k/n*100:5.1f}%')
full = (df.task_index == 'task_7').sum()
print(f' --> full 3-field (task_7): {full}/{n} = {full/n*100:.1f}% [C2 threshold >= 30%]')
by_nfield = collections.Counter(len(ASK[t]) for t in df.task_index)
for k in sorted(by_nfield):
print(f' tasks asking {k} field(s): {by_nfield[k]:3d} {by_nfield[k]/n*100:5.1f}%')
# ================= M2 =================
print('\n===== M2: ground-truth vocabulary =====')
print('-- reason (ALL systems) --')
rc = df.gt_reason.value_counts()
for v, c in rc.items():
print(f' {c:4d} {c/n*100:5.1f}% {v}')
print(f' --> reason mode share (global): {rc.iloc[0]/n*100:.1f}% [C3 threshold <= 40%]')
print('-- reason mode share per system --')
for s, g in df.groupby('sysname'):
vc = g.gt_reason.value_counts()
print(f' {s:22s} n={len(g):3d} distinct={len(vc):2d} mode="{vc.index[0]}" share={vc.iloc[0]/len(g)*100:.1f}%')
print('-- component per system --')
for s, g in df.groupby('sysname'):
vc = g.gt_comp.value_counts()
print(f' {s:22s} distinct={len(vc):3d} mode="{vc.index[0]}" share={vc.iloc[0]/len(g)*100:.1f}%')
# ================= M6: reproduce published baseline =================
print('\n===== M6: reproduce published baseline =====')
strict = (df.score == 1.0).mean()
print(f' RCA-agent strict accuracy (score==1.0), all 335: {strict*100:.2f}% [paper: 11.34%]')
print(f' RCA-agent mean partial score : {df.score.mean()*100:.2f}%')
for s, g in df.groupby('sysname'):
print(f' {s:22s} strict={((g.score==1.0).mean())*100:5.2f}% partial={g.score.mean()*100:5.2f}%')
print(' strict accuracy by task:')
for t in sorted(ASK):
g = df[df.task_index == t]
print(f' {t} ({"+".join(ASK[t]):22s}) strict={((g.score==1.0).mean())*100:5.1f}% n={len(g)}')
# ================= M5: time bottleneck =================
print('\n===== M5: is datetime the bottleneck? =====')
def fieldset(x):
return set() if pd.isna(x) else {ln.strip() for ln in str(x).split('\n') if ln.strip()}
df['passed_set'] = df.passed.map(fieldset)
df['failed_set'] = df.failed.map(fieldset)
# field-level accuracy, per field kind
acc = {}
for kind, col in [('comp', 'gt_comp'), ('reason', 'gt_reason')]:
sub = df[df.task_index.map(lambda t: kind in ASK[t])]
ok = sum(r[col] in r.passed_set for _, r in sub.iterrows())
acc[kind] = (ok, len(sub))
sub_t = df[df.task_index.map(lambda t: 'time' in ASK[t])]
ok_t = sum(str(r.gt_dt) in r.passed_set or r.gt_dt.strftime('%Y-%m-%d %H:%M:%S') in r.passed_set
for _, r in sub_t.iterrows())
acc['time'] = (ok_t, len(sub_t))
for k, (o, m) in acc.items():
print(f' field "{k:6s}" accuracy: {o}/{m} = {o/m*100:5.1f}%')
multi = df[df.task_index.map(lambda t: 'time' in ASK[t] and len(ASK[t]) > 1)]
nonzero_fail = multi[multi.score < 1.0]
only_time_failed = sum(
1 for _, r in nonzero_fail.iterrows()
if len(r.failed_set) == 1 and (str(r.gt_dt) in r.failed_set or r.gt_dt.strftime('%Y-%m-%d %H:%M:%S') in r.failed_set))
print(f' multi-field tasks that include time: n={len(multi)}, of which imperfect={len(nonzero_fail)}')
print(f' ... imperfect ONLY because of datetime: {only_time_failed}/{len(nonzero_fail)} = '
f'{only_time_failed/max(len(nonzero_fail),1)*100:.1f}% [C4/metric rule threshold 50%]')
# ================= M3: B1 constant baseline =================
print('\n===== M3: B1 constant-prediction baseline =====')
mode_reason = df.groupby('sysname').gt_reason.agg(lambda x: x.value_counts().index[0]).to_dict()
mode_comp = df.groupby('sysname').gt_comp.agg(lambda x: x.value_counts().index[0]).to_dict()
def b1_score(r, time_strategy='mid'):
got = 0
fields = ASK[r.task_index]
for f in fields:
if f == 'reason':
got += (r.gt_reason == mode_reason[r.sysname])
elif f == 'comp':
got += (r.gt_comp == mode_comp[r.sysname])
else:
if pd.isna(r.win_start):
continue
if time_strategy == 'mid':
guess = r.win_start + (r.win_end - r.win_start) / 2
elif time_strategy == 'start':
guess = r.win_start
else:
guess = r.win_end
got += abs((r.gt_dt - guess).total_seconds()) <= 60
return got / len(fields)
for strat in ['mid', 'start', 'end']:
sc = df.apply(lambda r: b1_score(r, strat), axis=1)
print(f' B1(time={strat:5s}) strict={(sc == 1.0).mean()*100:5.2f}% partial={sc.mean()*100:5.2f}%'
f' [C1 threshold: strict < 5.7%]')
# best-case adversarial constant: try every (reason, comp) combo per system, time=best strategy
best = 0
best_desc = None
for strat in ['mid', 'start', 'end']:
reasons = {s: g.gt_reason.unique() for s, g in df.groupby('sysname')}
comps = {s: g.gt_comp.unique() for s, g in df.groupby('sysname')}
# optimise per system independently on strict score
tot_ok, tot_n = 0, 0
desc = {}
for s, g in df.groupby('sysname'):
bs, bcombo = -1, None
for rr, cc in itertools.product(reasons[s], comps[s]):
ok = 0
for _, r in g.iterrows():
got = 0
fs = ASK[r.task_index]
for f in fs:
if f == 'reason': got += (r.gt_reason == rr)
elif f == 'comp': got += (r.gt_comp == cc)
else:
if pd.isna(r.win_start): continue
guess = {'mid': r.win_start + (r.win_end - r.win_start)/2,
'start': r.win_start, 'end': r.win_end}[strat]
got += abs((r.gt_dt - guess).total_seconds()) <= 60
ok += (got == len(fs))
if ok > bs: bs, bcombo = ok, (rr, cc)
tot_ok += bs; tot_n += len(g); desc[s] = bcombo
if tot_ok / tot_n > best:
best, best_desc = tot_ok / tot_n, (strat, desc)
print(f' B1-oracle (best constant per system, chosen WITH ground truth): strict={best*100:.2f}%')
print(f' strategy={best_desc[0]}, per-system choice={best_desc[1]}')
# ================= M4: B2 uniform random =================
print('\n===== M4: B2 uniform-random expected accuracy =====')
tot = 0.0
for _, r in df.iterrows():
s = r.sysname
nr = df[df.sysname == s].gt_reason.nunique()
nc = df[df.sysname == s].gt_comp.nunique()
p = 1.0
for f in ASK[r.task_index]:
if f == 'reason': p *= 1 / nr
elif f == 'comp': p *= 1 / nc
else:
span = (r.win_end - r.win_start).total_seconds() / 60 if not pd.isna(r.win_start) else 30
p *= min(3.0 / (span + 1), 1.0)
tot += p
print(f' B2 expected strict accuracy: {tot/len(df)*100:.2f}%')
# ---- ground truth time position within window ----
print('\n===== extra: where does the root-cause time sit vs the question window? =====')
off = ((df.gt_dt - df.win_start).dt.total_seconds() / 60).dropna()
print(f' offset from window start (min): min={off.min():.0f} p25={off.quantile(.25):.0f} '
f'median={off.median():.0f} p75={off.quantile(.75):.0f} max={off.max():.0f}')
span = ((df.win_end - df.win_start).dt.total_seconds() / 60).dropna()
print(f' window span (min): {sorted(span.unique())}')
print(f' ground truth INSIDE window: {((off >= 0) & (off <= span)).mean()*100:.1f}%')
Thanks for releasing OpenRCA — the committed
rca/archive/agent-*.csvfiles made itpossible to audit the benchmark without downloading the 68 GB telemetry, which I
appreciate.
While evaluating whether OpenRCA could serve as an external benchmark for a
root-cause-analysis method I work on, I measured a trivial constant-prediction
baseline. I think the number is worth reporting alongside the headline result.
Summary
A predictor that reads no telemetry at all — it always answers the most frequent
(component, reason)for that system and guesses the midpoint of the question's30-minute window — scores 7.46% strict accuracy. The reported RCA-agent result is
11.34%.
The difference does not reach significance at α = 0.05 (paired exact McNemar,
p = 0.0596, 41 discordant pairs). I want to be careful here: p = 0.0596 is a
borderline value and this is not evidence that the two are equal — it means the
present sample cannot separate them. The part I think is robust regardless of the
p-value is simply that a zero-information baseline lands at 7.46% and the paper does
not report one.
Method
No telemetry download.
rca/archive/agent-{Bank,Telecom,Market-cloudbed-1,Market-cloudbed-2}.csv(222 KB total) contain all 335 cases with
instruction,groundtruth,passed,failed,score,task_index.Validation of my parsing: recomputing strict accuracy (
score == 1.0) over all 335rows reproduces 11.34% (38/335), matching the reported figure exactly. I only
trusted the remaining numbers after that check passed.
The baseline uses leave-one-out mode estimation (the mode is computed from the other
cases of the same system, never from the test case itself), so there is no label leakage
into the baseline.
Numbers
Per system — the sign flips on two of the four:
Per field — this seems to localize where the difficulty is:
reasoncomponentdatetimeSo the agent does carry real signal on what kind of fault it is (roughly 2× the prior),
but on which component it is close to the prior. Because strict accuracy multiplies the
asked fields, this shows up most sharply on
task_7:task_7(the only task that asks for all three fields, n = 43): RCA-agent 0.00%,constant baseline 0.00%. Both score zero.
For context on task shape: 44.5% of cases ask for a single field, 42.7% ask for two, and
12.8% ask for all three.
Why I think this is worth reporting
With n = 335 and a baseline near 7%, the minimum detectable difference at α = 0.05 and
80% power is about 5.7 pp. The reported 3.88 pp improvement sits below that. Readers
comparing future systems on OpenRCA would benefit from knowing where the zero-information
floor is, so that small differences are not over-interpreted.
This is a suggestion about reporting, not a claim that the benchmark is invalid — the
per-field numbers above suggest the
reasonsub-task does discriminate.Reproduction
t0a.py- task structure, vocabularies, baselines, reproduction of 11.34%t0a_stats.py- leave-one-out baseline, paired exact McNemar, powerInput SHA-256 (as downloaded 2026-07-29):
Caveats on my own analysis
task_indexasks from theinstructiontext; the exactreproduction of 11.34% is my only validation of that inference.
for the baseline, which makes the baseline slightly understated.
Happy to open a PR adding the baseline to the evaluation script if that would be useful.