From 407d89fb6c02594e5bccef765b966e673ba73ca2 Mon Sep 17 00:00:00 2001 From: naindev <193035803+Nain9Dev@users.noreply.github.com> Date: Thu, 15 Oct 2026 12:55:08 +0200 Subject: [PATCH] feat(report): measure how long each job waits for a runner The report only had the wait for a run's first job, which hides the queue that builds up later in a run: a job that needs another one is created when its dependency ends, and on 15/10/2026 a seal job waited 13 minutes for a runner after everything else had finished. Read the jobs from the REST API (created_at to started_at) for the sampled runs and add a section with the median, p90 and maximum wait, the job that waits longest, and the runners online. --- report/__pycache__/ci_report.cpython-312.pyc | Bin 0 -> 32459 bytes report/ci_report.py | 83 ++++++++++++++++- tests/test_ci_report.py | 90 +++++++++++++++++++ 3 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 report/__pycache__/ci_report.cpython-312.pyc diff --git a/report/__pycache__/ci_report.cpython-312.pyc b/report/__pycache__/ci_report.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2d5d36100b13024ffedb8efe061e394ed3a33547 GIT binary patch literal 32459 zcmd7533OCfnkM*O?E9vDK?(r^1!!g8u@L(r3<$7gV-e_uLK&oKE|EyleoIN$~8UH6G6zSa1=d*r|NW%&zU(#wwzsQ&&)aV z{r4>qAwssh`kXoQK=<8u_vOF;{qO((_m6*Pw_7+oPtC;p{-lHB{*FGBM=yE!1&xN| zZgB$F$qBrm>E|zNIyF4`wf)*oEqm)a_3UlvG_bd^)5PB9PBVL3IxXyN?XHD3RU7fDW?oK!RHuQTg7jzcDZ|p1-Or1r7xwBZXbe0IUg7sxxXQ{AC zu;E>%;SO+u{bf#YyoYwu(^(#Hu}~*MT}tSCQY6<|A*>eM$XO}W2_C$c2|l3!?6T6uvRF-+bh%y#dy~UC4yEc)o_93N*RK-M{nkS$UmV$D0^Aoxk7GHIa*ZlM2l9M zIiWGPG@(hTM7~;~Sy+ZPtP)y;D!f+<>x62&>yTr8ZVvhqHV9ti@(DFBbDe91jly!I zT`Q~?HVG>bULR->R$i*(IFWi|>w8`vlB^fP>y=)<$4e1BH>w@g`ZoK=v>bOf5V+Jo z>TNsZy%_NKhc9}?z|f!=-sbJ`2m6P`fLHW~1Ktax-fsVJC{XVeh7k}P90=8WuM7u< z1Kx1(a=_~!5WIeGXuv-dx;Pm24)%D%gMxpw(PHu985;FAtoL>YyQH!k>AQja-ocT9 zfY>zPM-e$8gW-z-5kV*;D%mJ;7dYW8?E-(H|Umo(c^$_Jsxql+5d!;Ts$d z4-JRqt~Fj3ES9HHJWCu54~nDp-odK~K&9THfEXMUws}!7(;~l zhfppctd+G;>WS2ekQdEfG^T?Cfbx-xXbJUeRLMi5!g`8&{oNN)3XK@R67>a!LYBJo zrvhQ`@DL5(`Fg-OWf%!!oea=eA_<@X!z|4sn`btp7y$2(80hH__Flwrw+@VkF9rvC zy@7trKT^}QHd?-*1?l6{R{sFzTUuA`_%^lq`}_SD`U5^q+AR3P{_cK%D3sRs1Oxp7 z;&o5^!xtmZP|O8?C~!!qSHN{}pq@r;I8?6yWjhvEeK&fCxor(otSI@0hmfCzVMGSP z^;rMH9uaVH(Wi-6GTn~Y+6FHV^#?E&=zOGNM@>TmfKluXG_Xatjrj#Evt3_wqCed- znqkPs02f$>Uw9RsTU=O;>ywc?OC>u|u20^ivO}@4XcQFNXUvX~{DM|NY|6XO{A4Nx z2?{zvU(TK4e1^!WRxC{5IxLWZ-henH1_v1G0wU`cgF{#Yw7f!uoM-OmgLx1@Q?xBB65Ps5h>FKlRuPoFkk^oK6?2QQ=z0FiYYH>HhO zKK=ggK-v%(AXrG7F9xm&!CtHmn%{8XS~x^w<^6Wk-fhp1grDyYHb~Ib(1WRLz$Zh$ z4YceT+7l^F2E(RpR7MwIJp^nm7NV?L{Dro{8R!1q>bbG&`mWf@x9e}$PhFi3C(Ad? zm2Xd!Z@+&aS-v-E-8W}Fp0FNIT2I7Jo{pb79k-r|>&`47bME}oe6acLhbMb}!|MF9 z`9m6d#`v)8uggZb09r(JC)_qBHH>uNvVb@ovE z-s5LQsGvD;R}Qs1qs1PNFNw8 zAOOFso96l6i|4T>*a&(1g9CxaZgu+6Z1STY`~r>tEiTGOxi0vmXOz3k6VB6hVF)qO zx=^6MM_h);oB=jBUu)QVS9xZ8>?my=(uW zeMk46O6w>^THim2`MIkTSF_@1ju1wLq_u$F5H*^}FbPt180JvKDi0!kY(aeJ5F8Ak z!*e5eJ^0vFm~s`RT*WC@X^Q-1k6Z`%Y^ppa?VwFBXT_w zTk*$hrwWtKRdLg*Cntc8C2Rs>@O<|KRZ?5;R>Q{Kuz1|( zoNE%!HPf4B)+L>r;-*bcP690wHVJ`j6f*!R z?xZ}s;N${d+KS@4&=*iWp@>ID1X*G2l;tvLQ z-F|=5{kr&`j(FASxa$l+vcNIX_O<2vK=Sur>p55XoU=aRtpAGRQFKAk&kxOO;rlk^ zMvX63v>LdN4MnY0+K-p`FeSDj(OoS=@eI32SOu$@1E@00E*_wA92vw;r96wU zkN^jBsnM!)8S9wZIK6*n!_2DqmV@y%hvJoolb$1(!lkgv)|m@Q=hnDs>-SG#X{(m| zSXw2Wyodqh5T5mhEq%>mqQ zlZTx4TyZTqgxVvfh6d@tlr}JeT42aNQkZ?VQVvgI19GG_8=J)@crY8ZD`zb@hQza{ zByE+-B+@3c)Uv|Pqkzy6I4atIV(TqudoZ@)RoI5H8PAv4X!CWjO`vM=ip#tCs)PO` z)b!n(Z)VUlL!3ncVIs1o;fHR)7G>fv4OVFy(wZSLGYdrZ&@A9MBpoK%NJ}$OIxtM^ zfO;tcCs?90ab69vN$}~UNyy!c#SJvBIyhJyGTmd53g{XVLlL*UNHRepWjGEecOpRg zWN08Z5m}T_%C+VnF&Qc)G=SQ^4~fxfm2JcZ+KAuM^{S}p6!!yAkCx}2eqv#)5OEjj0qbyBeGlXgEn|w0x(>5gT zg@XVhKPAdXHw6AfC}$;@d%zIuqBCj4C6B%u@-i}pPQf80Wi^+k3QBHmy}9*qVQH$c zEJg0lRB>gjBes8P&2-1~{&?Z~d8@(gSA+{yD$l|8MAsU@o%|#HaC@))B`{g8mX2`oRqX>nOv=kx-GANCUz};!$#* z#6Kn~iGjRwYElAzfed*t&mw@AfZxY3Ptgwp_I96!(a(``g^Bgtb9BOa4u_X>=jy>U z6VLe!(jLZE?I!B77Q7b_%mJL%2fnqi^g^&ZoSSktb&^<2;&bFY4+ra?Xl;oW&xB@| z4>?Fy{I?b~z#H|pgK_I~q;aey?sHdH+S1i^c~BSzkaneQU0qj({rz%`v8ziM?C$Df zhkLP(ilZYFBge#jp(8_xTKzKjneOQ0t-HVG zczx-D!J%Kh;MM6jFWAg*iuL+!3l0~;Rx-z?Fa5d_KB+`1ryGCjqm_L8!UddcGkk*6 zkn04rf@TMz1#PEJ(BVrjf9ctmLH;tZFQfcrWM3xsWd%Y;sBqOKF$C ztnAAnf7#fVQ~t8EFBkhVcRB<&O7X~XP8L@nf4SIKp`6mqQWnWy9`;o%rz~J8OXRph z*0NIhtB8G-$zPB*p}unZf>fj=P;KUEt7^_U~PltlC~+|7luQlX{Yiv*aIdsqAfDn0T%tY z7UFa&H4XL);5q`}Aa)K$$MoOeCmE`3;4f6*M-rpi^LVP z(TSxCtWxlag|wRv6C{M_lADaiszF0ro2l{Y_htXJt`PV?P9f0aAMOuhNV-WT6RBia zAUupB33t?c*#Ompu^&EnE^P}z;6m-DnsZprGzC}<`~u-pM(xNmnf(&WKzdTjMGrz3 zh0tfj%HiB8;7=qbDll9CFR=&CU7qcG60=s~u$XC5yGU*pDNtdgQ4M^6Tp=%6S%i*AN8@7CKg6VByR9q&GO=edNlIj4IVN`|$A$l_ZjzJQlV zyg~6ioV2s6D@!s046F{j*h`66D`~>SOXSd*LcB~4(N^LhIakObTr7&@gvbe#Gfd7^ zaz@CxMvj4;wd6$Lq)lK3;}jkq6~`#-$K+ffr-&Tl`-+0%{4;n$JSOW8xW^XzPYl&4^P|3uqQkEe=?#}7?}lDgtldBu49WLr{Kky^DTt}B|1CUt94D^?@$ zXi~R2E4s97{OIJ0q^>MgQ8|8m@>Eh+nN{-2I?5bL>grO3MdJr2x|6!1R9X4>vB~vG zUHRkc70A3Msauh%sYTI;le*f+nF90HthV9vmQ4@Bvn@x*kH$TXNo-Sk`&SxXf0$p; zat6md^UfRK%?kaJg-WAC1Cz5XX>8fD{BPi)Wx1ET!FOwjsdKK3Bm+@CKT}pg(t?IW z-68(f(kOpZ^J;~T8`B_{Cd7|wGlvzhzqzPpFQ*`DCAX*%^jN~$Q>c%}-xxnaGN7ZY z8St+DFBc7h{SCH`I1+?FZ#9W^EOYaY-oQZM z+K{*_vVsW^gFU0lhSRvCf3Vx%AKKNZCi?eiOo$-+f9HNb&c(|%zi*gsdDA!5`gZ;M zhHnY#lj$mCQy&Gi zu5uzjs#~HYeTKi81?ZLE0q82fR4`@$UC>}{Z~*rz1>9{M+GgbeVb*OAsUskQi1WEEc~hc*?=AaM&xlrRzy)7C@n`%kuZo$A>4^eLY< zt;HUj=0O7xNMqM$CFIPp_%kYroCN+tgf4NAHhLyQw?=M`#IC;f`DTJg}jGZvk0z5U}~t&LajjO%s+6c8PqV5Cqz^MhK@(Bv2~#`f#m=d3Fe)|FFdlGggTuKr0< zfIyvfVjGIQ1>QJNAyDeM*8pLfs3tFTs~AN>r6Qry{3%s#{}S;!WdO!>W7?=z)I@ba z%)2vaII5<|3R6psfC-XsV3n0M2Bt?oQU4OHQJEt0sx^d3(!rjn9&$H=a6CQg>IOFb zBOT_cl~%2t@FsD69P?JZMBlNBv98$Q_b{+9bk zlj{#nmP{Bwa~9w3MUqDaWwF(-?SALX^npL^N)&9E^Ry*AZAs6*31iA+zhS#>n-p$c zym>KpHCepsVe#hahK~yFpMBu}b#=Vx(5I%u^G?oPI`86a?(w4%=*v@x=5d@6e+=|# z8zD*p6ab&mEXl0SY(BA|$;b;c3{5M5Afl(EaRfD3Raqc_I!jm(&0=(B@+x2@KwF06 z2o(dIbdmb@0JNz>;RdMETq1R{ZmfHt&Fzfkz#s^!Ls$L%(1dDi|AG+07Xln8A{au@ zMhOTa(UNIxe_#M}9UPztDS28u)E!P62StI@qtZI6VKJ98Q=#|)iuphAC$o7zr!BPL zk)v?(%qs)q2Oe1+6IWh86)Sn|`N?BbZHdCw)9Vw3jd4#?(%L+}H)Zv_zJ4Pro8r2rCy|kgQQt&$|8*FZ(SZc2iyojDELWffb!lf3WMOd3#Ny|k ziuN!+nn+!%m(VoGH__{7UM2@5v|Y~xgQPrmx}&X;uy!~CVrLNSsy{}l;v3{p7sU_B z0l|i8LQ))JJQVO2oE!?w#xOkiN`(8xUnBZ=l!Q25&<(LWZ#;MXxjFl)gniXiIBBmR z-=8X~xOMIq=O#AAb^E3bzjWMne01SAJs77JW)o#ig<{ zchs!zL4y7&9UMwnDdi}|F=h)BiNGEx8>zI{m@`b$5-HrFv?}WCBBaBfuv_s#qlTll zD7#ixY86PM4(!Qir8WvW>ZxG5v>E;5U$sS@*Z96drL@djFh|gaZcUduyI~>?*n?T2 zLKZ!k6`NpT<0vCar7eP0rX|~0Z5Y|bk6N4B?^mD2-iUUqv)e%J&zUn@_MGL!*t28U zKEG^njhJmY2y@zm*>GV_U1E8-T1naGReYIu)TQ(Zv!Tub=pJ{tR!J3g2o7juJMX#P zBb0%@{U%C(gXCl zPRG4^1pO}R!#R8Lqd*^Kx%5U!x0bOWtf;#*T8ftF;46ui+|<5mKq+>>XW3XOLdwKz zS-7E0sSV?$uBz{W;OsmVN~^+K6<@SWC{&0Fl&Y?q@Xkz%tXkF8l9?ggbD%!AD(N%t zOrI4W?m}?)N|gG>%A@6iTPQ8(P;-rv5A(QB@n_x{&ICp)h|#KuR^(kDYJY`t!b0LM zhJ*HJN>Y3(4qBmvu#^XtugrUinp93c*gWO|UZ|8Y=t?E$tEX9shn4TlTdiBgBClQo zcaV2UNZJD-5tx>ZAF#~+m zI#z{tJ*(u&yrDtqRnlRVRm->(5Ia^Kt;)h`)k017exdIE25I*P{;ZJq|7zgX8mvX{ zSanuUy;(g~u}@Y{m!qfNyghAHS{rq$Toc=f!avF9^T@HA53hUtSOSGD$+cC1Sl|_AmY|UtnajY))Dj~T!dq&A#LNE|@U=sl zLdXsT^#opNUd$$^#h)WAQr;%w(Nz+~mjM6m%((n(o=QK&3uKz0jGGk2Lm1aV`MUAJfD#~WxHx~^w z7a4AH$-29nru=UjVprZa-`(_$_zH^3Y4ei9DE>PnmBcRpBQ*l$${aNDe}eZr25&u* zivON+Q#(WxoSZ7@l*!oe0z4{I*U5wk4>npBS0V4ANj=)+<1O@PWq4nc%edDvK2R7c z5kMHE@I=?T#P`t#aA75|A4v1Px%}1d8}8jS+=RU)(6=4_aM$}AW?E+a6aLrB=iJK^ z?&VW0Nq6l#q3M=Cy*g`e{BW1d(r;|$b4G;-X9mwWS!Ka9I3=9KjPB9k9Ib5^X~8h2 z)jw)-o}vb$9zb?1Jk@%8BzAFHH+|x+F|n+f%@^>{|BUD_s6BKMgDy3t4YI6TM>1|! zc~0Ba8GRm^{)kuSbOm9m>zd3F7#kGh*eKfvBd4jRMbCTkw>*=zm`R>tE)84Sr0YVg zI`L28sIzw^tpz%l=Wkq+6MonSA zbL~jDcFehU{f+(`#`yWEub%&62eRc11D#bFU=w3R9Rxtf7`nZbm@PwHW5?EXYc66h zJcp8qn=DQIFEp>3bqF2PfHKet3V1hQP;|1~K{LW6RJieER@Dd621$8|u{;^nw2S{1 z4O7Q!2x14O-b53Ced~vfSO<+*8*;D#o9A>@hR@xF6Rjz0@r?h2!25yw2OnIR-E=5f zemJq|(Ddop#;M{rcE&dyN+7^`7@Ejt%V+MwZ_?$5zmb&0%D3=u+SZ$V7h6fp$S$w#{wTW63@=}# zEr#KErow=mJj0hIvZse4{s}qT$+<`lVH5FFa(+urA2~DR{24j#kaLNgd*sjo4LeF8 zEYp(WpHt{CIZV6Z1M-cK^Uuh+1_vTs$O=MC$wAUx>!+y8O^2Bu-Rf1zmz);U<{bZrl6@@YNOegc0TnofG?8Q~tqmz4Dc97a(BEqTwAv(zKW zRB%m@5vQUVaP@yt=GSz<>CkKSa#q*R>nGQbx20^(*O%X_zga)Ech=)e*nH#rAkFj? z-YU6S5-W@i&$??D^g0``SuJ5R+l`IaH%?Y1t=_4F53ThxMIWtt;C`_EP;6FPY%c zmWk)D?~Vl@T33@Ch$J~bx5fB`sq7DJkE|83J#QblePC+ubj|e9yG_Z;ZAt6)`y)x~ z{*=us?6Ri`MUTKf7+ykkz*$Jt$$MThdvw0pBmL!OrO_r=n7OtDv zhu+;ddi`juHoMK`Tjt6RB+3pX%MN{NJ3Q}3)#$Iu{;1kFSKXYbZk{PhR&S=}Rz9v> zJ-&a*=Hf<1)#@qn-5=fg(Ts4v^?v_{$CIm%{&v~1>&GVcPGTNBbGBs(+p?H2wI^v? zJ!fl7*czutKDBNBQp;7f^YcbjJ#XPw_}*=})9_BytnI*^rdZLxDVpj1`Pm0W6KAKI z#`h&`2R<{}9xd}u6}{0(jVjL!>%QdjZERT4^y%6ePqKOgYxbk+#U+Lg2&{>;_b|k< z`nJi7No)1gmWS50xuEi?&C3F+9@?s=uD%<&6PZ~xyL#)Ln)_E0wx@n;t9ne3RrAnR zGj$?iTlHI84Iryv#k_-qY^D8rdu&5ef$fd2MHp;TKbO}&vN@APV!=t;+T$lX61ENi z!3qoODcTGn{MzelCxxV?V!Z89QE9B;wP&yp+8>oK`%)$~ut=*63!JXPKn0h~+c=YL z;>Coq{Ij*2Ue`|Ui*28Et-il+LjSm=?AF-Lv8j&f^|K|U<^WhLX_~B=+4#}^2Q3de zKi-+#abkA;Gx5{U{C52_KW~kfkfMVOay0)npj@7#mX%}BzvQGnYC(_p0%!2DHdmt0 zBrW~st6>dR|2JP*I8O)v%~z#ZC*OQkj4HobsN(Dg5pP`D#KrGHNyQLid`qy8WN_;lP$VLtr2cFGOCYE-|!0u5ww)n zF*Iv+M579N_#|y(BQ%js_%gF_BknKLmxaDYjXrDI+Ck<#4l;`*;t)d9*8aiXF5HGA z1yv$Y2K#~YAy$Q4jkL%~%W7z}T|7sAcW?mP0o{yK?!Tu!?5-U%_MwD9s`mm?XVeYF zar$%tbBnN9^MtCOmBPp&-HK2n(*6PN$8qKY%&49eQta-Z9!)Xbzbw7HT3=jO9J9>o ze9|eD3h2h3pgu&bV;K2`9XJAK-=NQuV>7M-DWQ%*-3an#^63Tbm?5gqM`F^3M-SBL z)td6%kjk}{|$Xj8==|+3awWpNCB78#B5j~AmU+3o3gW~jl%<% za2_7Ofu7bDR8)pVT&oQ;dJZN|oFXj&7{a&)mPjZgX_u6!h@!?~zB~lO03iR;Ca8R0 zrafle~~tW?rpGAHWc{%#1py<~2o7?9K>JtU^)2Dv<%-v_+>zr$ND$(%NUoQXaRUfZP z795%|J#x4w*T+h4?wa#7Cp^tb&$^il3D1_KW9x)AWp~AN6I~DOJEsa}SMN;J>>S_! z7+dkq>pNfF{m5A!TR&x(HYc6y;-+#M6%XTMSPobSI&n??c zTdX6#Vsp~91p!7^+*qpSe(>z9^LX5J{E?$Pwl(RfC5@Bfl*u}AX394`@^Ja4$!8xt z8-MyNu&2!c>Pl}&?PyyR!cb<;;dc?XB$Exjqy?N;ZC}tbvD5M;4#TQ?0E{!}^D8?j zD;}CBTs{ms?pdO2RiOaN3rCQbmcJDD7Siv{N^R|Te9;b z)uI;Br=&0-F4ZraG9sw|*b*h!*{C-x(K-imS)QyIy>lWa&;EiK*ITNJAL?~4QL-w+ z{vq?}^R!1CD^_@q(M1*sO)`#$Sbzl?+1cPdq|gv#>}QnS@Lup7n-BO3x&gW*wc#LNow_YO8Pd=zPH zV3xZwt&~weG*e+F2cjy)v=GDojbbT{z=5phQiq?&EtyAv#EXn}^P%1sZ)u(TUbE9tauaz-1*HeG^K~ zP;TH086-SQc2EXj&L>N9fqPP5F~GJ0n?i9da{dB;Av+xCEZdjG#JFww_`a0U{<9Z< z^5SIhp|M(4T8tNU%$iO^iSEYe_0d>S(peohRU_S;u`FRMiyLbc<;Jd0O|>br^Tx8D zFO!2(4&U^uxivcyYj(taJLfrm*KRoWr}*D#wPp*>XJ!YfMaE0kPVb+q-<7D}6<@pi z{wUNTXHAFmHf7(Zru|e$#r2Aq@u691BW`9m%Twjvx$#czZoR&B@^sQ#5!Y2biF}E0VWjGh;2}}h_)?0OxXb|^ zV~jfVxX6x(_ALkv5T;{cPKqV+4l#&_AB=m!@CJCwZ|S6Ta;%Z^uvnI!rO{IwvY^8^ zFmi;+rh+ceo{VRc);6yb()>lxC3>)Y1_bcviKtBLnL2GQDD+$ot5@tqk{8`0P#_){ zsjoZ6f1Gl99@*TJt6ym+h_FqFFGW(GlF5=7Kk;JRR8B$Rm!b%Yt-HB)av7NQ`f1G_ z%k=tqb<^~fxU(g0YDqZ?Cd1b+PjtmvW5ZM7+tC?KqH5jD`uMWVGgso)t#RGf?-=WP zWvpTI2_z%hCq3*S0A8?ykd;yP>DomSzKOp?x-?%aHsb@jjCx6jl=4d)c3513I;prB zAs^r`v>Y+$HG0=Du^yWA_w>_&neMxnl6BkStGC~8iPw^Tz0^{I1mz(joixq$7y=2ci?W z3U*n6C7#s35dUQqR#XJff6J4ejy%G5X@rG>d;! zB>czBl{Zte;-gs%XOA4MlSPkG>l0~LE=q{cCDX3@?zB;pEVC(=;i~S*^oYrw4V*RE zNS9=f^dx@{O!y_CxfCOvhg^mE302&UBe=0IPyHbq!q+_6E^*ruvr(469Q30cF@*XS zJ&38uUk73;-%^k2j7-cA0jhQs>jm*~7Ck?4300^gPYmj)mM{cZ0QpPzA}2AV7Cmop zQ8Pv`i+o=qe+}}lTa+1}sM&~^4S8dhXOqWBMIx3(&rjAaQQwOE=~3TG#E|f1(L-iP z6c{E}OkRwbm12^Ab*qjOYV)n@sCrDoj=rj0{0-`gm};MGsJ+}!D?~gip0G>IX!MBb zY1v9|L`ODlztwo_BDN(r8|^Hd20`Dt34Jm&=moYvd>1nbPrmlZt+37Us3OVc2Q3DNG6n{I8orLrfds1 zii1avIbWoeLzpPa8;!I?VN$gL;c&P8pjSi_^+vQv8qs*8Imc$uT9V9~07#V?V{c&! zT#IIyK4qz}2U6keJExLd@gWjq(@~OQR+&Us=h4eb5qUu&`}5yjxt#gSh_%5=Dw|EIX00EsYy3?0yu*`In=p4sd_GWUZHiC9BG?jy#BSkQkuL zHj)BfmJwt>g+R;)69RNtq2r8nB}2OB6=K&uMhTK~qnJ!FAi#VttBKvS@@Cl(mP+-) z5`km^*?%c8fIDP^bZmp&U)YuHC3DWqTrio_l+wdi9W&T~dnT78g~7|D6G)|mwoCS; z$%Gxu9t_|#2mV}8fZ<~$?-0&BuuzQiAhlI#K|s73?3NB`I8HT6<9KQjY#yddYbuyg zZ}0_k3>4161{6>vpoLEH+!Nb^2e5@mEeR0}6iJQDz?b=221 z-<5k;>ieMfKN8f^CZMP(cpabZf9CyjGadIgJaA8Lm|PXpyt!jqm#k=rmp4xPXI;&S z{m;nYTwTM9|AHDoq&PHP7+_lg*r#M0=~9Sp1%e{h24$SL(dk_VkGhD-OKMIqmJOe5 z#v`&OEA0Ro9~T3?;F~gg*>y0i1beLwFfxv7WIhyCCmHwfM)brJ@I@RA(#oQv_XsL0 zZtxz5g&=Q^QYpd%k@Pn8zgp#;3c!7L0If27-Z*YCD$yq~Hk*Xt9Vx_eJjWcE*B}7zw zhN&yDbYZWEI0@n&WcxM#LK^|WkoDU#vVObk^>w#)+}ttMFXQM=~*9lZM+J; zv~=FX=`9cKD-*hv(4Vk-)DfNxV_5g!-XGgC*)#F`&gH_I9WDG&Gs zb5)Irs>b-Trs>i9?%z~>SOtTOpP1q&PR^aUkT`K6j`8lk6nFLiPOCw+Cb`G)vVF6* z{TKk3Te7YacQyhroTV{-u5@jpbZs2h?IG4*_t;%9ZzNf=!eN^H(Ukw){yY6sFU;(p z*%B|>F>BhX0>(;cige!ToH{zA!5#eF@9&PUYm2v?joY4qoocYDp#ES-R}$b4kqQXY zdq{a2r?1R4?M^i9jyFE_K=F_X59VXq=L-{f5~|9gRW7WafAXE)`*3Dm z{J>;+M%2lFPk!}$pD88lCI|E_!|JKO&yz``_yjG<$-x9KR6;WEzLJbz@nx27-`Los=7N+Dxga3K z#RaAuFBKLsDdB0SnzoCD`D~J?4@TFR?Y}P4Uk?rTlZ7l|??NVM4qm3J$W*>=759eqZABPVX}%%wk!hIP=FMQjkp7&0#PER zhfM~u_m-n*owiUmsk!1QfSw7YbLd56BBS}v*G;eNnzPmp#Dl>*$+rnB~&j;m1jf9(%#WzF?2QG#n6RmbyXuq zfdeqnHTe*|crl|V(wB!pn_; z{+!BaL`^VELE9o8C5#S@47RA&gDfh?5zmK@C;{s1I>9AWh3;Ik|OhtLTq$ztpeW}gBwcOIsGDnX3-l%uBRFp7Qkjn4}LO^&#cX;YWRS{<*Ppi^-8Q2S7IC5*`^M;qZQUg+0QA_2231% zCcKXnXMXx^JJFZWa9}@prG=d%wppi*@`Z-9?qZ;ShzzyEHWOHCk$Oq_f`}05x85Xr zgME{h0sS%)s7bG15WofD#pki>NAUY9#2@~Gwp>sRy6`CYiqBDurJ+H&wUE}d9YO{c z43p&S!a`&ehkS{;QGOk+o|wPhx^>O#HZ`;~H?(XZTQZrij+T~f&CR~;UgdT#crA_I z)36grs&|9bTM?8e_;r3S{d^Zw1M&BOhXbw)x!tg-ieG|iWJaFFe~lJL*iS?%~nTd)VWq#vX6GvxtH89^TSUHAdPNIg@_ zQOn1Xh!lR9ZVb%~s$oAOMr{CrY9(MV#CSmwfnb&Kz9OsGFM@!7+aDm+9b(}rKYd6B ziIjF?<>=^;^aZUza5EXFigp9>8*;v-P@aWGG!{$5q0S!r^+QB`PRWRr^VO$KezBLC zB1$`2#ol4EM)WkZJtz8+01!b0HPe(y+b|7XNX`<;mW)|e3JgicQE8jQX)AzxHqw&J z`V9nHUt$#j$xiakP*Ob`74aAi1^uKJSY{bDPpP5*iW1Z2FLm_S;x%iMasStyr+EG zi&FOUa>-g`G-YKhp=!71tv0(qR`;B>GGV2Yn|1m4-oLlGCJS$s-7I_MI9PF&)o<(@ zKaw(9fA-=_FFu007Ji=U&6Tn6+b`aJFKiD(7?m)8mU}D{YY5d$x?96Lp@pT6#tnuQ5;F7&w9J40fD-))b6Z+4ru16+& z+_CASQ@`o_u=7*X!IabW_ZA!78@bVRz3I*6Z?C$&DoN_QkIc?jO2_x4>gs=K`R^^S z9*q^gU4FZK*0%D#4q8PUw#;qV`*6cva8$!{)lG@&rddz(Co9K~d|A#Fl`oXp?S?N) z;hWg|bqQy7rEHFr4X*13t^m>N(Dg&OQU<*p`kfwJtx1)x!2kbTU;)Ff82RSQz^0t% zP|VkrxV|)5i|Wg6mrb=z56@aR!qOYY=|?a9C}pxu8s4mYr~g+Q?(2Tz_|TDDf8Z0( zC#RCDP9)1tCR`^Irjr;bB#yik!Hd44Ht+K8z;8C_jn`xasM-KTAN(Y;WS^b%UoVe|j?s;;ubNo{T05#w@3Z!5= zRxk>Bi_)pZe`^)OjkITx`~$K7T~1MvlDrnq&(V(_{5euatSp|HJwGS@it=`n!BSeoE&-up%Iq~K+5Q`k)UF6L)ecO{mMkoF#Ac$E)l362E?hjH^}$LDTW$8KMxb-4BsUSw50ZQ3ERW_O7{MMC`NLjMUa#7R8= zrN++depkiuEB=mi{w-JZx158y*1zT4?7!tda4Y{;uI@i@rH}2s%!~W7Yv&7?w~+I!lqvv< zH#iqcSm+B<7mWOxc@B=;0s5@u+)eZ5a^3*t<#{dMXbvhil22+4`7E5ly8LF8f|WwQ>eLu{&qAS| uul(9|jOXq1=QX&YJP~=QElcTa<841X_>+UzJ6<{Ssjd)ojINxqyZ>L`6~m4I literal 0 HcmV?d00001 diff --git a/report/ci_report.py b/report/ci_report.py index 6873cfd..25838dc 100644 --- a/report/ci_report.py +++ b/report/ci_report.py @@ -24,7 +24,7 @@ import subprocess import sys from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path RUN_FIELDS = "databaseId,conclusion,status,workflowName,createdAt,startedAt,updatedAt,event,headBranch" @@ -75,6 +75,17 @@ def jobs(self, repo: str, run_id: int) -> list[dict]: ) return json.loads(out or "{}").get("jobs", []) + def job_timings(self, repo: str, run_id: int) -> list[dict]: + """The jobs of a run with `created_at` (when each became ready) and `started_at`: the + difference is the time it waited for a runner. `gh run view` does not give the former.""" + try: + out = self._run( + ["gh", "api", f"repos/{repo}/actions/runs/{run_id}/jobs?per_page=100"] + ) + except subprocess.CalledProcessError: + return [] # the run is gone or unreadable: no figure is better than a wrong one + return json.loads(out or "{}").get("jobs", []) + def runners(self, repo: str) -> list[dict]: try: out = self._run(["gh", "api", f"repos/{repo}/actions/runners"]) @@ -170,6 +181,13 @@ class RepoStats: # Only for repositories served by the shared merge train; None elsewhere. merged_total: int | None = None merged_by_hand: int | None = None + # Time each job of the sampled runs waited for a runner (created_at to started_at). + jobs_sampled: int = 0 + job_wait_median_s: float | None = None + job_wait_p90_s: float | None = None + job_wait_max_s: float | None = None + # (job name, median wait in seconds, jobs seen), the three that waited longest. + slowest_jobs: list[tuple[str, float, int]] = field(default_factory=list) @property def fail_rate(self) -> float | None: @@ -201,6 +219,29 @@ def _parse(moment: str) -> dt.datetime: return dt.datetime.fromisoformat(moment) +def job_waits(timings: list[dict]) -> list[tuple[str, float]]: + """(job name, seconds it waited for a runner) for the jobs that actually started.""" + waits = [] + for item in timings: + if item.get("conclusion") == "skipped": + continue + created, started = item.get("created_at"), item.get("started_at") + if not created or not started: + continue + seconds = (_parse(started) - _parse(created)).total_seconds() + waits.append((item.get("name") or "unnamed job", max(seconds, 0))) + return waits + + +def percentile(values: list[float], pct: int) -> float | None: + """Nearest-rank percentile; None for no values.""" + if not values: + return None + ordered = sorted(values) + rank = max(1, -(-len(ordered) * pct // 100)) # ceil without floats + return ordered[min(rank, len(ordered)) - 1] + + def parse_moment(text: str) -> dt.datetime: """A date, or a date and time, read as UTC.""" moment = dt.datetime.fromisoformat(text) @@ -262,6 +303,24 @@ def collect( if waits: stats.median_queue_s = statistics.median(waits) + by_job: dict[str, list[float]] = {} + for r in sorted(conclusive, key=lambda item: item["createdAt"], reverse=True)[ + :sample + ]: + for name, seconds in job_waits(gh.job_timings(repo, r["databaseId"])): + by_job.setdefault(name, []).append(seconds) + every = [seconds for group in by_job.values() for seconds in group] + if every: + stats.jobs_sampled = len(every) + stats.job_wait_median_s = statistics.median(every) + stats.job_wait_p90_s = percentile(every, 90) + stats.job_wait_max_s = max(every) + ranked = sorted( + ((n, statistics.median(g), len(g)) for n, g in by_job.items()), + key=lambda item: (-item[1], item[0]), + ) + stats.slowest_jobs = ranked[:3] + for runner in gh.runners(repo): if runner.get("status") == "online": stats.runners_online += 1 @@ -410,6 +469,28 @@ def render( f"| {s.repo} | {s.conclusive} | {s.failed} | {_pct(s.fail_rate)} | {_pct(s.infra_share)}" f" | {fmt_duration(s.median_duration_s)} | {fmt_duration(s.median_queue_s)} |" ) + if any(s.jobs_sampled for s in stats): + lines += [ + "", + "## Queue time per job", + "", + "Time each job of the most recent runs waited for a runner, from the moment it was " + "ready to the moment it started. A job waits when every runner slot is taken, so a " + "high figure with few runners online asks for more slots; one with plenty of idle " + "runners points at the runner service instead.", + "", + "| Repository | Jobs | Median wait | p90 | Max | Slowest to start | Runners online |", + "| --- | ---: | ---: | ---: | ---: | --- | ---: |", + ] + for s in stats: + slowest = ", ".join( + f"{name} ({fmt_duration(wait)})" for name, wait, _ in s.slowest_jobs[:1] + ) + lines.append( + f"| {s.repo} | {s.jobs_sampled} | {fmt_duration(s.job_wait_median_s)}" + f" | {fmt_duration(s.job_wait_p90_s)} | {fmt_duration(s.job_wait_max_s)}" + f" | {slowest or '-'} | {s.runners_online} |" + ) lines += ["", "## Snapshot", ""] for s in stats: lines.append(f"### {s.repo}") diff --git a/tests/test_ci_report.py b/tests/test_ci_report.py index 38c950a..2b229cc 100644 --- a/tests/test_ci_report.py +++ b/tests/test_ci_report.py @@ -68,6 +68,9 @@ def jobs(self, repo, run_id): out.append(job(name, conclusion, started, failed_step)) return out + def job_timings(self, repo, run_id): + return getattr(self, "timings", {}).get(run_id, []) + def runners(self, repo): return self._runners @@ -414,6 +417,93 @@ def test_durations_are_formatted_for_people(self): self.assertEqual(report.fmt_duration(3720), "1h02m") +def timing(name, queued_s, conclusion="success", started=True): + """A job from the REST API that waited `queued_s` seconds for a runner.""" + created = NOW - dt.timedelta(hours=1) + began = created + dt.timedelta(seconds=queued_s) + stamp = "%Y-%m-%dT%H:%M:%SZ" + return { + "name": name, + "conclusion": conclusion, + "created_at": created.strftime(stamp), + "started_at": began.strftime(stamp) if started else None, + } + + +class JobQueueTime(unittest.TestCase): + def test_a_job_waits_from_its_creation_to_its_start(self): + waits = report.job_waits([timing("Verify", 45), timing("Seal", 780)]) + self.assertEqual(waits, [("Verify", 45), ("Seal", 780)]) + + def test_skipped_and_never_started_jobs_do_not_count(self): + jobs = [timing("A", 0, "skipped"), timing("B", 0, None, started=False)] + self.assertEqual(report.job_waits(jobs), []) + + def test_a_clock_skew_never_gives_a_negative_wait(self): + self.assertEqual(report.job_waits([timing("A", -3)]), [("A", 0)]) + + def test_the_percentile_is_the_nearest_rank(self): + values = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100] + self.assertEqual(report.percentile(values, 90), 90) + self.assertEqual(report.percentile(values, 50), 50) + self.assertEqual(report.percentile([7], 90), 7) + self.assertIsNone(report.percentile([], 90)) + + def test_the_report_gives_median_p90_max_and_the_slowest_jobs(self): + gh = sample_github() + gh.timings = { + 1: [timing("Verify", 10), timing("Seal", 780)], + 2: [timing("Verify", 20), timing("Seal", 600)], + 3: [timing("Verify", 30), timing("Seal", 900)], + } + stats = report.collect(gh, REPO, "CI", days=7, now=NOW) + self.assertEqual(stats.jobs_sampled, 6) + self.assertEqual(stats.job_wait_max_s, 900) + self.assertEqual(stats.job_wait_p90_s, 900) + self.assertEqual(stats.job_wait_median_s, 315) # 10, 20, 30, 600, 780, 900 + self.assertEqual(stats.slowest_jobs[0], ("Seal", 780, 3)) + + def test_a_repository_without_timings_has_no_queue_figures(self): + stats = report.collect(sample_github(), REPO, "CI", days=7, now=NOW) + self.assertEqual(stats.jobs_sampled, 0) + self.assertIsNone(stats.job_wait_median_s) + self.assertEqual(stats.slowest_jobs, []) + + def test_the_section_names_the_slowest_job_and_the_runner_count(self): + gh = sample_github() + gh.timings = {1: [timing("Verify", 10), timing("Seal", 780)]} + stats = report.collect(gh, REPO, "CI", days=7, now=NOW) + text = report.render([stats], days=7, now=NOW) + self.assertIn("## Queue time per job", text) + self.assertIn( + f"| {REPO} | 2 | 6m35s | 13m00s | 13m00s | Seal (13m00s) | 1 |", text + ) + + def test_the_section_is_left_out_when_no_repository_has_timings(self): + stats = report.collect(sample_github(), REPO, "CI", days=7, now=NOW) + self.assertNotIn( + "## Queue time per job", report.render([stats], days=7, now=NOW) + ) + + def test_the_real_command_reads_the_jobs_of_the_run_with_their_creation_time(self): + calls = [] + + def fake(command): + calls.append(command) + return json.dumps({"jobs": [timing("Seal", 5)]}) + + jobs = report.Github(fake).job_timings(REPO, 42) + self.assertEqual(jobs[0]["name"], "Seal") + self.assertEqual(calls[0][:2], ["gh", "api"]) + self.assertIn(f"repos/{REPO}/actions/runs/42/jobs", " ".join(calls[0])) + + def test_a_run_whose_jobs_cannot_be_read_gives_no_timings(self): + def refuse(command): + raise subprocess.CalledProcessError(1, command, stderr="HTTP 404") + + self.assertEqual(report.Github(refuse).job_timings(REPO, 42), []) + + class RealCommands(unittest.TestCase): def setUp(self): self.commands = []