From 51a8412ae1ab064a66c46f9923cf85e52774b0e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mats=20T=C3=B6rnberg?= Date: Sun, 1 Nov 2026 11:40:53 +0100 Subject: [PATCH] Remove Needle; stop paging on nginx 4xx Needle scored ~chance on the golden set and never gated alerts, so drop the sidecar, proxy handlers, UI buttons, k8s container and build step. The labelled eval sets move to cmd/triage-eval/data. nginx 4xx (registry 401 challenges, /oauth2/auth, scanner 405s) is now dropped in FilterActionable instead of only being discounted, so it cannot ride along with a real 5xx. nginx 5xx still alerts. Co-Authored-By: Claude Sonnet 5.5 --- build.sh | 4 - cmd/needle-sidecar/Dockerfile | 28 -- .../__pycache__/server.cpython-314.pyc | Bin 15553 -> 0 bytes .../eval/__pycache__/rules.cpython-314.pyc | Bin 3012 -> 0 bytes .../eval/__pycache__/run_eval.cpython-314.pyc | Bin 6936 -> 0 bytes cmd/needle-sidecar/eval/rules.py | 42 --- cmd/needle-sidecar/eval/run_eval.py | 79 ------ cmd/needle-sidecar/server.py | 240 ------------------ cmd/server/main.go | 2 - cmd/spike-alerts/main.go | 56 +--- .../eval => triage-eval/data}/golden.jsonl | 0 .../eval => triage-eval/data}/holdout.jsonl | 0 docs/ML.md | 2 +- docs/STATUS.md | 35 +-- index.html | 36 --- internal/alerts/filter.go | 9 +- internal/collector/handlers.go | 105 -------- k8s/deployment.yaml | 36 --- otel-ui/src/components/IssuesVisualizer.tsx | 92 ------- otel-ui/src/components/NeedleTestModal.tsx | 236 ----------------- scripts/run_needle_test.sh | 21 -- scripts/test_needle_classifier.py | 168 ------------ 22 files changed, 29 insertions(+), 1162 deletions(-) delete mode 100644 cmd/needle-sidecar/Dockerfile delete mode 100644 cmd/needle-sidecar/__pycache__/server.cpython-314.pyc delete mode 100644 cmd/needle-sidecar/eval/__pycache__/rules.cpython-314.pyc delete mode 100644 cmd/needle-sidecar/eval/__pycache__/run_eval.cpython-314.pyc delete mode 100644 cmd/needle-sidecar/eval/rules.py delete mode 100755 cmd/needle-sidecar/eval/run_eval.py delete mode 100755 cmd/needle-sidecar/server.py rename cmd/{needle-sidecar/eval => triage-eval/data}/golden.jsonl (100%) rename cmd/{needle-sidecar/eval => triage-eval/data}/holdout.jsonl (100%) delete mode 100644 otel-ui/src/components/NeedleTestModal.tsx delete mode 100755 scripts/run_needle_test.sh delete mode 100755 scripts/test_needle_classifier.py diff --git a/build.sh b/build.sh index 38c3514..773d2ac 100755 --- a/build.sh +++ b/build.sh @@ -5,10 +5,6 @@ RELEASE="${RELEASE:-$(git rev-parse --short HEAD)}" PUBLIC_SOURCEMAPS="${PUBLIC_SOURCEMAPS:-false}" UPLOAD_SOURCEMAPS="${UPLOAD_SOURCEMAPS:-true}" -echo "Building Needle 3 sidecar image..." -docker build -t matst80/needle-sidecar:latest cmd/needle-sidecar -docker push matst80/needle-sidecar:latest - echo "Building otel-debug receiver image..." docker build \ --build-arg "APP_RELEASE=${RELEASE}" \ diff --git a/cmd/needle-sidecar/Dockerfile b/cmd/needle-sidecar/Dockerfile deleted file mode 100644 index 8c7f6e8..0000000 --- a/cmd/needle-sidecar/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM python:3.12-slim - -# Create non-root user with UID 1000 matching Kubernetes pod securityContext -RUN groupadd -g 1000 appuser && useradd -u 1000 -g appuser -m -d /app appuser - -WORKDIR /app - -ENV HOME=/app -ENV NEEDLE_PORT=5005 -ENV NEEDLE_HOST=0.0.0.0 -ENV NEEDLE_TELEMETRY=0 - -# Install dependencies -RUN pip install --no-cache-dir cactus-needle - -# Pre-cache model and shared library into /app/.cache as user 1000 -RUN python -c "import needle; needle.Needle()" - -COPY server.py /app/server.py - -# Ensure UID 1000 owns /app and all caches -RUN chown -R appuser:appuser /app - -USER appuser - -EXPOSE 5005 - -CMD ["python", "/app/server.py"] diff --git a/cmd/needle-sidecar/__pycache__/server.cpython-314.pyc b/cmd/needle-sidecar/__pycache__/server.cpython-314.pyc deleted file mode 100644 index 8b783da9e968364720cf53f8dba7af341b43abcf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 15553 zcmeG@TWlLwb~EJgB~lV4QSaB#vMtM|sE1#&{E!vd@vkOS%30J0DuojH)D8$cETWN{8;*#?j$ z0O`tsEZ+dK6d=oTAS;GU1MOKn>Q8QpE7xhoRqQlu2vwPZs(K7mbq1>DF;KM`sJh2M z)n}mC$3VF=P@8rdyk@zIYk+UxjNdL-KPCTe;kN4hr;jSvv`}&_*9bdg+eNCyAlD7m z1J0WC_IhfF891zCGh4c;41cATLe` zAwI7=6yoEu#GV02R)~h6IV!VKVthO(3cSPyqhU57!9!0lBqyZip4fOiA@eO!GRaO> zj(cYufNHlmiWd*J{T6FqsM)|0bB+z zs%6gvB_7{7{)GfD$;V;Z2-MsAj~+d8>S$o#4CfQRTS8Hpn&W2%eMw8Jm;4JM8PcSn z%*b>p9`eD##u-J{BLp zLTbEhcRU_J?zub$Qv(iI4QH%8#Js^QOq2t;gOoy3Is8dAkX@zj zo2mTLso3n5`NoAkw|Bi?t2`T0>|uooi%tOe(wGaz(vrqpq6OZ_>eak3ALpa6^FmkD zd|nh|Vqlz?B-pQS)1n=~nrJZ}3XQSIrx7M1SuHj+LZ)%j1mUC!`qob5pb7fD9<5v& z7(`Z@ptX;BdR9tyv*Fm~D6l&p97j4O;_2t~{$K@(KzM+-BMAv98j1n!h$N6aASYrH z@FDU*y5qK^XL_4FIF`uE2{8(-nL!iL;6zY}pak)X#qfD*Sr$uyL=;vL8pBqN0g#@m ziQrCj!L!a7STB;zmctmeX~x)KNW>{Jhg1()nnRQ_@4Rhc@9k3x=T|QB%BZZ^6AF{~ zzhV-T7z4qpn&HExSX8Wpl|7CvHsSKK0%nx3#VCD?u)`?*o)Ot%l>P_86{BC6EB>mR z4Fw}8M4oU%J5l}b*%n+4@5D`!z z5E&?IVvz{)4nTAdoWOyCz#gL@i9n0R6OjmeS&+v77i=UpDnz}k4;V^YL!-ow1QQVu zMj&gU8+ic%13U-3nP$^8H7=OdE~Z z0OPpfPpX3~!%Z#ol?%sipL?HIzB;4?c*Q=VFe8sL69avW@b{pVbpHJ6ti=Etz?hXX zn%I02+n^vxuM_oBA7Pt5`X*tUKKd5XM}28_$*4={lvcaU1Kg{*`iRIoQ*!ipP#lkd zTw})p(@0{GmtqMqgyaJoBg_t?4Z+4DVHP;x63UVB80ZMlIT8;<4TCX*nk*C>A4iP? ztP$oXfV*OZVbY2$P2WOLIT1bpqKFqku%_usR0kocA*(nf{0?1Zl<6vCO?N~LZYK$a zNrs4xbCvVQ7S7)GDfW{JbMm)IKAiW7yWWE<)YuHk=Og6v(Km@Q@X_xQ^7+0H`CjX0 zhZCdPZUPCe*+j78Kqml2!h9!K2~i;gdryb}T{4LGz(yY<_FNctKU$k$7^2z_^DnZv zjbkIM7>thcNZ?>N5(@$4$*xCxU_^|K69Ev6utBscKvqD1mI%W%sSDDJ350>zAz33V zif?W6rYTL7z-mgD>Xg<#Men@>I^th0p|{T2=J&#W9a8Lr3N!d9q0{{Th0IQvO-qkh z23CEG5FPeJ^0&b-tAu`xkUvJ>BEmifLr$cN9 z<`SZKwj7C}9RO9A`C)cK0I4mdIi?uQ4;X?E2O{7<_QTIuImK&l3WFS=+~Z zVZAwI!HOKRl%&a$)3+0#@J~WhUq8)7XNCFlg{Ir*R?#C_@tps%cSrNkj>GT(h2M)6 z@Q}0|;K$*xaVS~sAKup0=?{Oc&9n0*zqjdYkf&dUlgt4jekj@DAAYX2d0()3q`SFy z=xZG>HA9cpHM?GF#(t}-+S*_84=2qB5(y!EC|T>j6c59LX5zD~=171q4}(WSdSwq% zZQ8WQj&PnnQ=0Rv4@9UzPuze$wGII#2h-`)tKSF)Tw70sVK`G3hBL$BEXitgZ%XL3 zjLKpx(%c=1#4b0V5d}Cns`U0`&ARQUc{pE&rDXB3qdpHi0G>uLN(Kn4Y|!I47_Pi1 zC0lSJI88SDuEfEf1?Srs2*D)LbRPEE<})%6ix?#kcqZ-#^4Y=b77l*>?0{HyM8Q0RZ$j z8Nu;~XaVy#9*cr~e+H^mdlplYcp7SQ@Dz>((o|ZDr)-OWowU76kWjT~t(w`nWUYs{ z>xI_}m#mc;BBxH^5RiHQ8q%GMY8swo6+PadR%kmSW)z{qE&ggnyR@vGN? zAN!lEBBEKu>t1maX2|Q}cFbBZ!|f*SgRCiE^9K!L1#T|!6lOliRC55=CaJ~<59wD|`56>&@y-QnqR~*h|N8O^MZbn*iG{D>SGuO^6IciCL z&7z}b=AzQvwd6RQsaSHY{B z{G=7#b<{m7Jdid-c>Dmw$cDnB4e4{#6uyMQ!JlDeU|I$m)}(beK134e8((L7vMurU zWK9~t>xDHbyY1qJYU->3U0;~OU}3|GPy~`RkP!uX4f;9H8uU3&(=Alo=rgXnMi4Uk z8V4G({q-kZFT_nglRp1Zdh{@^z)Qu=8;ok#=hpj#!7)%n`OE`b^pUeA=kOUh`)VW6 zLA7UAUr%i?M@H;?t~m-|j{K*dqwuM|!wGW~KJ^?$Pc=s|%u)2zbCf*Q94?rnLgHev|MPwVrC@R`U@xfLD%dlFs44XVd_mUDu@5 z@i_!$9Tk`}#;U`PKGXnSWUWc7_AfdI)+C5OFmV+vS;gyK0|2j_MaPC-g5tF{Q2r(z z)@RaN;wC?8ACd+<`ct-aQ$EwcI&rjIj#>T6BXVd55ltKPJptnp>m8@Kwb{Evwg zL!S>xT;+ble#2^xxJW}_MkXbU2?J`;7RqM@L#K+XMuR7A1FEfK(yTVy0OEIu?mk-v z*Xz(@Lz-vtSo8Vt=-0_c4WJKmP0B{K#Pj?edgB24veqPhn0jgz+`%_ZuZxl}Jb>ox($)ZL%DKM6(w73f5hNo7<$15-;3foeitj>GU}>vw@6(*_^qnTX&YtI=7sr zxh?S`ZYx*KH3F{PzCv!BuZU~%6>!`AJ^FWTu#WBA4t=fqc${YgSWnhs;F|qMGb1}L z8uT$ZFW1syi`)Fi^wz9o(8u>5*V{VY1Dgh95>O^9Kl=36Z0X0NCP{v@l(nZ!4}ILV zCCF@Bc3htYWyt@_4Ac`~$*vcG$q6GoRvY3hEai8mi3OL_;a8E)ab|rA z{7!3YvyVZ$_33qXEokjH)5RdRv?qZUsQt!TN}!wCKeM&yk$uSm{V;oqkB&kx#S{%^ zMR0LY;(5%##-@@5C!$*D$H~Dn{he^73zhwvPgM(ZO4$MTRQl0MBpP~Njw7mR5LlxhJ7MRkY0@1gUAvAf|SrTtXkmk3K1j{2{|Gs zWwl@=5hXE9a1xG0B$bH<$9dHbonU1FbiP7}pms^6<_S>oRIYXLkL2)!O4jw;A z%?rh%BN$*A;zwa_l2n@(&jVgxH4m=DFa|6zE{Q|XDqh5F7_ugV7=t3|TJ31isnfJg zJIdp^Otol^L^!;U^0I0Y$)O!ixJFUH?0M1? z(G-Wkyg0_Hfd?QaM2SBf<_t<>5n&iGITmc+y+^gd9Yt6eg+NY`$NnMg&)}V~>WK5= z2*ge%z)vcwCct16m&rqbhiViPQNSRChJgP8kfbtUpo?l6fi)u#bP9`M9Ltt*qpQYv zEDj{W>j?%t6Jk@b=7q2Wve6YAKqm$>91CB8;ev8drFo4hP4S?JPL50b6)}N>8sSb+ zwLq|-3=OatUJ_r%)nCR&$A}zX4sPzlQ^U#tWiU9Q@FJ@LpBzO!&>n_ae)6-o-kv($aOiC{@>!syZ^+ zw^C9*-Fc&Py6Z;Q?EX|q3%D-~*8K91cC@@MPbXhZ&Td{RZ=C0*nES<)t75w2M#oI= z>@%s7#^sXc#gb;y?bo3`=afgMyI$?mhuS^on&aQ8p37IPBLcQ3h~St+lXPTok)Zb_AIn_}SWSDdb8=cYyHrrDiK&PI5fF1%5=He+$*k}_P*+!b-h*n!|ET^rixpZi}x)S?@JZ$pX|9;T0MQ?#)Vnq zYk|q$mBP|Hh3?O=zEUhsSfai&C(w_+$l%RFUqQx%Np;LHKxj%CXd~76kYGT*7u#$ z_lnD>n{G7C9GNSfYJv}f@2IL@YDcQ7dEWA2)!v0&A69lvp8UA1a(eW}=!`H| zoGSB79{Z@N_1z1%E$`F+nD@bv<=tPov->MEduQ8z(5aj|KY41U)V*A~WwCV2?AUxk zs!-7}_{;GbFFbbQY-dv>Y9vv6?o1jO?eRn6>L zan{U)SHKClb;VgWV?$4%AL`!VXM5f{_QPXyrxqGgn-8Vxy0B&N&2!&7H~ZZDrc~{2 z4d~=NovLfsT!;5d@=6LOdmdI$&XU*neEVx}2up>HKpNzMH+J1*Zl3*-BURnJYa6SI!p)p)moMzF=2qi32}k?kU6pmnKk0uRf_m zodN1{PVZm~WxO@ummq=s$)e|?QwpBBYwaPfViNn%-(**<0isR<0W~iWfa`z+#4!f&Hv}&vf)P|S z@KzWI5RC+pk9;%+KPn(T{I9~SVg$3dFnb3wwKPE7`9dfVltF_JLlCh90~7@U#DG8* zi_tiN0N^2iRV8#oY8=+W%}?NXn4vD0wn6rrSKtFbH5g41`qMn4sru7Am#O|!&js34 z|LYeG2GbE5;!GgdzD@569kpbJ0t_zd!7m|OP5JDPF#L=LN>MuveuO7UKkLOQKXMOA z9F^^be3I=6+`wed72wUL*7JC0eF?}faB@IfiX=EWuy}F+FGF^=T;E<{8gx~W9S>}5 z7%^j2Gs!KFaN;TQBMZFthq%TuS&nzMw5W7vlFb_&OUMxBG)ltiS&f6)GzY`oKVJ9X zW>OjaHH^yOuVe^~#Gha-+@QkmPXZ8ijX%Lqi~9I0!?B+61rsn$C-#0L(x)p1|vbAQ>T65Q0_nUk6nuov=b@yOb6tpc_+evHnqP2S2T6^CF zEuTq6FvDLvyDijDEnBEFvH`gSc_Q$W{sHb zzzom*;%>}3AyX|7j2wrHF%f+`#2gl<@hM~X5?YAY5!N8Ls%A}Wh*JoJn?l3zU*eMs zGd#0u;>LPF+xv&ae}X#Pi_%3P_kANx)1NR#nt6~%(Rn|o;J=;xxBZ+d_=xKHIhFq} zmb}UK>z&s+uOGU0Xr_72m9l$NmX@o`ipBOy^5x_!zyI>@PmRx>Cw2E7l)?H+^~=={ z4fsQ(2S&<}f1f&G`Wm{soE&bV+{>QaA zD6{pI3x9Y)aqdW&JadA=bgY;xT8%elYMC!vUDKB`oqB&vVa{b>T2rRB`MnCWKU33~ zGHsh{RG9WmO;gIWeeSrz?7p9GDKuQQ|7S7AIXS3&=ad1>g6V=A1xkI}{H~O)^eQ`biTlp(Y z#R}tIVK%KWRVz&83RAbj)UGh)_ls<`hKH2RX1r?p#6`w{Th8TL@10t2s7&-6`o!GEJca>SL7IhwT^7bY$~!Sl-B*X97&1lR#6yOX#qyWaCbt3!dFaw(kJXnO1$N~BfAa?uPh zZ#Zw>_ukt#^WNht-HAAY@vDFQ=Gq^k2>o4cS|fb;_CxB-hFg^DPMuumW`mmzN4WEs3zwKIVMYPS)PBDR!jgxh}B;^<0yY z0*Kp^cm?TtCa+i}v*d9hD?S&D;Igc^cm-1j=N%&R%v&a|=~JIbaEu5uUp0L~SnZ0S zW#UzYvJ^d^o7~J`^Juo0{(EeskRxlD+U9lAo~3KbB% zY~FHsHZO%JK^L5Gi-pz5^*5yx94EEx^Lf~<)^vxBfY47BAy z%LUpBSJ^aXsu6rIUGmB#UB-f^;bo_mzviY~BGN@~!V|j_V)PA@mtATU7to!-^-7ps4G!_<1y$!|8)%pFq+r!YbbAZlx<(Y zVms`Ca`Zt$T0|MQ@DUfnO?(1%*0rj+iVGDH zY8X^~x&fkSl51DvAiNA^JLo5CKG-C7 z&Y@tuYJs0!D)Tvru28a?4AFlnC7&tsZy?FoCx3*Heex$o{tSdpsH2o3Ur|H_h&~3A z)PpdGZV`l_Lnh0tMb__X-vKPAZ9txOKG9X(y$$X$p^uaI@*tQ zZ-`r6^{M?t?`Gefp{=3KwO@?v^qp_?oxeA=ofxan?sg?^&)l5Z9KZY3?XENRncdFr z+ZS(M+{zkcuX_R#o;eZL+0Xz0W0$1iUuzw>dWF*vi`Ia{CJH@ep44Y@h9 zW1N0qoZjm>vv0)LYd30}=XQ+Y2gdMT&&cOE=RU_7-!&4O13&%AUe5*fYviF}?)5xn zjeWP>zy&LasPVN||9vG+Y&Njy8AHFo-c>BkVcb6W%z0|Pm9jkQTDmAFLZTXK7 zs%X>t6G$Og_IgJ5I$vZe*^`}tk;~bhmCFS>gBNOfWgq^W1lk-^^?^bALF`M4Kx;$n z%ZjLy-;D1^sHx;I^xrOfHgt#YvRC08f{ys_fjkOnnzo1D{447CTP%J%{_{9EGx)j= zbWXpbf2Nin>YpAgQ2#sT^?GDC8e6~i!L>%;a-;0+l&{|}U*C?hdZ^9MHL$gV$$d<= zqlF`U32NWhrj*+EwbztdwSZb@W-8%lCk|Gtt`3rNPU==JbF QqY*9fq<2_5-TcD;05PP&9{>OV diff --git a/cmd/needle-sidecar/eval/__pycache__/run_eval.cpython-314.pyc b/cmd/needle-sidecar/eval/__pycache__/run_eval.cpython-314.pyc deleted file mode 100644 index dea17073bd9cb4fe5887fa5ae2bd3a05c79f7a75..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6936 zcmb_BZEPDycC#NOmlR2nk}TP>Y^@~AifoDc{;({|mL)lkWy{pcrfi=qnkLs06N+5z zE-jmJ5#@^3v~!5)0D);FK zy=NOB8My2Tggr9P%9A`h#PhxY&q{tD@AZhRXTswTN;2zVMgFp%pW%J%L@40ngRI2M z4wLD$3I^0k++#ywSsglWG7WgVliHGEXr_yOv(oRYtYa(rD+2G8d9^R|SLDh++0S0k z1~0IF324r6;g$V>Ht3n+P5z)a5cc_l6KrJ=c$M61j8(FiLZT{xOP)aBqR0D&!!%S& zDu%vLI4JX?lIPy9Idh@bN1d8_WuOppAt%Q1 zQsI!cu^dsCGdIR)FHTj7wYRml0#wgMG!WXiPRqu5{H7@aM{#rm!Rb0M^-u)zJ7~{e zgUhX5%Oyg{K5b(epuX$j2)N_$(ywL4^*4Jic05pk zXn5JZn`IT7Y6v+Nq%ax}VhI3YsY`zb)ht>y+2$NK9FI)wl8Iem^5#r8OtF@C^5c6S zFx9IT``o#k=ivb8=opuCD_h|bf`_B}TtK(_P7 z&PT@bC1ZJlN^&V<`6FZFvaxZYcQwCo?&{5}ar-;heorA&6O6-O+SvG?k{KxZ1yg*u zR`-kAyu)2aFT7_8o>Xn&uZ;+K$nUa-tWy%O6>MS~92&z?Q;RdB`Pv9VPH^8Z7mnb_ z)^&~KC`~uZzikCNX9Z0EOU1c~){kX0nbZ9rR?>rRGv7&%ajZZifirP=oVf$53KW%t zEo=SUTC-&Ot)3A%D97xc+C>uKRDR>~mb~IL{!2l1t zw0^dA88}NFP@xq~{sMD`*;E?}xuOh=$+bDm7S5D`$s4`z1gX4SI$>9QstmYd!Q|3A zTV93$Prf#_G4t>74tt(QK>fpDlAzX{!kNFJamBM`5wK@-@rx zfiEmWqyJHt~b(^*}*jX{isSD(^ zkt;KopSv2@4m@_a%o}|K*9oKhV042oP)EC9bjt<|`@z1M?7D;9>wFT<%o(HUD9S{E zKJd;!c?QA>U%g!56rwkrh8gHAB-gF!$~eIt(4t^Sj|w0?f+q}dCipqh4>9N%cD!oQ zBi}K+zF^Feo%NJqqikSbZKsT|i}ctY?Mr8!*2}h>JsS_?KsvGzTCW0)XUlI(}`f=O_H@k?=a*X1~V4clK&~B04o|Q*0+s4IFn54Gi=jA8_~f_j8dgn5S-y>{8&xddT1)k(3~flGsUJPm_^3>l)DL-={0V#+S2 zm~#_G#T0;?Iq1FWo{|(sOFR|hI?eDY8G1v}1w24_gz9eURA`v;M#!EHMGqawGNIq8 z7^JWughUx$_IV+L%oH5;;TYrDVW_%^N{Ihv(J6EUyx$CXQ2dAm)6FAvj$g8Y#Zy<% ztPXs}y7%KChiaqz#5^%HkLGm>>1b34ULgd9xU3KX2?yPfhOo0DzN1i#-4lEexDk7l zk}1gj9o>Nt&?@yfG86bO(tL?=z z9IyYWrJQ)*R8IU6tTp|itclpL#Clv|!+D7%5uf!Uj&VBiWR`Gy?^1R99pfE++@2s7 zr$60&yW+iyband+u^j*-x}FY4^Cf6Hq1iw_c-OU^f#4Qp_5~uNX*-8<{oDFZKn8*R zFTQqBcY+kLHyKhXo+rfycoA`2FX4l6H>++$mg)m|1~wh3(zevxnTc1#=(&lT6CcdH zc!ss)`&&wgh{@#fdc&f}dzFn)j^^r!o()~*MK)5*g4Id;*(tCkPmo>LS`5^9Ng+Xb zDM|Dy#K8#3!eDn(lR~inpX`b_{94-`I8wfa$v!-jVUlgXUzHqAwB9TE%aXsXdLiOK zMZ}64VN6qgN#doh2-)2kF|nH69l(^gM|5yiBp(3QvDLyD?3_Z(fQ{?@L2wvyZMnjD z#0kM8O1xt26(_<|FzUpwqGI;=d~TRk$H00e#lQzI`$b6asR>>N`}N3^kne~5kk=m_ z;06@kB}ngO#iZiHpGpd?rv3UgN2uu3I|9)Pq=PGfs{l#Fd#vX5A{@_PZv$7nyntPt zq7y`aP=@We3Kt2I47U-YLi1PrvciCGbYqslNeOVR;DdQ^8Q^0mDLkd7cAijb@;V#3gP@s1rR#H51QPItSz$PS!IS|WU0LSu;vgT-L>|I0w z!0h%tt-xyn}ap;+~(3-XHzR*&m$vLI2#bo5y0) z@1D5ZGuxlaJGioK`-h_+oQymRc z-Eju$UpFo8{ke6~3f-lSGY^W+q%CJ>kF3(h=xA&tdLq6%Q4rsrIFr1XII&Q>cz9uF z%5-R%?)}127$sIL#c&)n#V$Xv?2MAD7TdDzP~x?FC-0tI-qn`29ZFjc!LZf7Z0}8+ zz30B`UaoFW+k4a2-YE5DX<1Z!cP3F4rN1oQ5xp2|d3Q3=5~WrdOUxc?i5cQ_{B-PE zqA#gS>`zJy&B@7>wc{bv`NRS&J-4Ae>k@X8a0iw))qGr)ocK}Q15^8KA0~Y;PR09T zR}=b=q-4imT}uQXBciQrwX8DH^1vY?$pGF@14GTI@z{R z1?S0P)l&6=x#{Tioth|>vi4v;NZ>7w2e)h8}1llP06BrGk0g^hNHuG46_3%bHkSgGhmFqvCNcyVK0vM zuA1$!!?Edj(fc!TF&0SdNe(9?KR&dG_)uW1chKG{$N^Y)ugN^mp}1D!sjKVfYc# z0f3D4#nWK|nGX?PKQ|-Ww&#ch?Kj&~Mb!y!a@P<2iSI3t3r!1Ese<;Ojr{!Fr{^A+ zkFJ{Y=Z@Yy8XKM)ygB$^k9F|Le?KioTiRZH-f`x|(=t@D|Ak`hdGS<_ERE6-_=jKG zN`_hbmvyC3|NZtI!`0-kcG$UY=2xxu5yJGFJ*6Xh=C=h~NAmQ)ZO}vgkM*_@8~snD z9_pp)JZ_(Esj-J^)qUQogTBJJ-PrxO-L-{^!R_{iyl%IM&mh%Ri$`$p6jTsR!dH|! diff --git a/cmd/needle-sidecar/eval/rules.py b/cmd/needle-sidecar/eval/rules.py deleted file mode 100644 index 9ea4a1c..0000000 --- a/cmd/needle-sidecar/eval/rules.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Deterministic baseline for the cause categories. Needle has to beat this.""" -import re - -PROBE = re.compile(r"(^|/)(\.env|\.git|wp-|wp/|wp$|wordpress|blog/wp|phpmyadmin|cgi-bin|\.aws|\.ssh|xmlrpc|userfiles|@fs/|etc/passwd)", re.I) -DEP = re.compile(r"dial|connection refused|eof|no such host|upstream|unavailable|broken pipe|reset by peer", re.I) -CAP = re.compile(r"timeout|deadline exceeded|pool|memory limit|oom|too many|queue full|exhaust", re.I) -BUG = re.compile(r"panic|nil pointer|index out of range|typeerror|referenceerror|deadlock|unhandled|chunkloaderror|segfault", re.I) -LIFE = re.compile(r"healthz|readyz|/ping|retry|cancel|keepalive|vital|\bcls\b|ttfb|fcp|lcp|snapshot|replay|unchanged|session\.load", re.I) - - -def fields(text): - d = {} - for part in text.split(" | "): - k, _, v = part.partition(": ") - d[k.lower()] = v - return d - - -def classify(text): - f = fields(text) - status = int(f.get("status") or 0) - route = f.get("route", "") - blob = " ".join([f.get("title", ""), f.get("message", ""), route]) - if PROBE.search(route): - return "scanner_probe" - if BUG.search(blob): - return "service_defect" - if CAP.search(blob): - return "capacity_saturation" - if DEP.search(blob): - return "dependency_unreachable" - if status == 401: - return "client_credential_issue" - if status in (502, 503, 504): - return "dependency_unreachable" - if status >= 500: - return "service_defect" - if status == 404: - return "stale_reference" - if LIFE.search(blob) or status in (200, 202, 204, 301, 302, 304, 308, 499) or (status == 0 and f.get("kind") in ("span", "log")): - return "expected_lifecycle" - return "needs_review" diff --git a/cmd/needle-sidecar/eval/run_eval.py b/cmd/needle-sidecar/eval/run_eval.py deleted file mode 100755 index b967013..0000000 --- a/cmd/needle-sidecar/eval/run_eval.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python3 -"""Evaluate the Needle sidecar against a reviewed golden set. - -Usage: run_eval.py [--url http://127.0.0.1:5005] [--golden eval/golden.jsonl] [--json out.json] - -Each golden row: {"id", "expected", "text"}; `expected` is a sidecar action name -including "needs_review" for the fallback. -""" -import argparse, json, os, sys, time, urllib.request -from collections import Counter, defaultdict - -HERE = os.path.dirname(os.path.abspath(__file__)) - - -def classify(url, text): - req = urllib.request.Request( - url.rstrip("/") + "/classify", - data=json.dumps({"text": text}).encode(), - headers={"Content-Type": "application/json"}, - ) - with urllib.request.urlopen(req, timeout=60) as r: - return json.load(r) - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("--url", default=os.environ.get("OTEL_NEEDLE_ADDR", "http://127.0.0.1:5005")) - ap.add_argument("--golden", default=os.path.join(HERE, "golden.jsonl")) - ap.add_argument("--json", help="write per-row results here") - ap.add_argument("--min-conf", type=float, default=0.0, - help="predictions below this confidence count as needs_review") - a = ap.parse_args() - - rows = [json.loads(l) for l in open(a.golden) if l.strip()] - results, confusion = [], Counter() - lat = [] - for row in rows: - try: - out = classify(a.url, row["text"]) - except Exception as e: - print(f"sidecar error on {row['id']}: {e}", file=sys.stderr) - sys.exit(2) - pred = out.get("action", "needs_review") - conf = out.get("confidence", 0.0) - if conf < a.min_conf: - pred = "needs_review" - lat.append(out.get("latency_ms", 0.0)) - confusion[(row["expected"], pred)] += 1 - results.append({**row, "predicted": pred, "confidence": conf, "ok": pred == row["expected"]}) - - labels = sorted({r["expected"] for r in results} | {r["predicted"] for r in results}) - print(f"{'label':<18}{'prec':>7}{'recall':>8}{'support':>9}") - for l in labels: - tp = confusion[(l, l)] - pp = sum(v for (e, p), v in confusion.items() if p == l) - sup = sum(v for (e, p), v in confusion.items() if e == l) - prec = tp / pp if pp else 0.0 - rec = tp / sup if sup else 0.0 - print(f"{l:<18}{prec:>7.2f}{rec:>8.2f}{sup:>9}") - - acc = sum(r["ok"] for r in results) / len(results) - print(f"\naccuracy {acc:.2%} over {len(results)} rows; " - f"median latency {sorted(lat)[len(lat)//2]:.0f}ms") - - for src in sorted({r.get("src", "?") for r in results}): - sub = [r for r in results if r.get("src", "?") == src] - print(f" {src:<11} {sum(r['ok'] for r in sub)}/{len(sub)}") - - wrong = [r for r in results if not r["ok"]] - if wrong: - print("\nmisses:") - for r in wrong: - print(f" {r['id']:<8} expected={r['expected']:<16} got={r['predicted']:<16} conf={r['confidence']:.2f}") - if a.json: - json.dump(results, open(a.json, "w"), indent=1) - - -if __name__ == "__main__": - main() diff --git a/cmd/needle-sidecar/server.py b/cmd/needle-sidecar/server.py deleted file mode 100755 index 7da5bff..0000000 --- a/cmd/needle-sidecar/server.py +++ /dev/null @@ -1,240 +0,0 @@ -#!/usr/bin/env python3 -""" -Needle 3 AI Telemetry Triage Sidecar Service -Accepts OTel incident summaries and uses Cactus-Compute/needle3 to classify and extract actionable insights. -""" - -import os -import sys -import json -import time -import re -import hashlib -from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler -import needle - -PORT = int(os.environ.get("NEEDLE_PORT", 5005)) -HOST = os.environ.get("NEEDLE_HOST", "0.0.0.0") - -# --- Cause-level triage tools --- -# These deliberately do not restate what the event already carries (status -# code, is_error, severity). Each one is a judgement about *why* it happened -# or *who should care*, which is what an alert reader needs. -@needle.tool -def report_scanner_probe(service: str, path: str, status_code: int): - """Automated scan or exploit crawler requesting sensitive or nonexistent paths such as .env, wp-admin, .git, phpmyadmin. Applies whatever status the probe received, including 500 or 502.""" - return {"action": "scanner_probe", "service": service, "path": path, "status_code": status_code} - -@needle.tool -def report_dependency_unreachable(service: str, dependency: str, error_message: str): - """A downstream or upstream service is down, refusing connections, closed the stream (EOF), or returned upstream unavailable.""" - return {"action": "dependency_unreachable", "service": service, "dependency": dependency, "error": error_message} - -@needle.tool -def report_client_credential_issue(service: str, route: str, reason: str): - """A caller lacks a valid or unexpired token, for example an unauthenticated client polling an API or a registry pull without login. The caller's fault, not a fault in this service.""" - return {"action": "client_credential_issue", "service": service, "route": route, "reason": reason} - -@needle.tool -def report_stale_reference(service: str, path: str): - """Harmless missing resource caused by an old link, removed asset, deleted commit, or dev tooling request.""" - return {"action": "stale_reference", "service": service, "path": path} - -@needle.tool -def report_service_defect(service: str, error_message: str): - """A bug in the service's own code: panic, nil dereference, unhandled exception, index out of range, deadlock, or a 500 from a real application route.""" - return {"action": "service_defect", "service": service, "error": error_message} - -@needle.tool -def report_capacity_saturation(service: str, resource: str, error_message: str): - """Resource exhaustion or overload: timeouts, connection pool exhausted, memory limit exceeded, OOM, queue full.""" - return {"action": "capacity_saturation", "service": service, "resource": resource, "error": error_message} - -@needle.tool -def report_expected_lifecycle(service: str, what: str): - """Normal behaviour that is not a problem: health and readiness checks, keepalive pings, retries, cancellations, client disconnects, redirects, successful requests, routine info logs, web vitals.""" - return {"action": "expected_lifecycle", "service": service, "what": what} - -@needle.tool -def report_needs_review(service: str, summary: str): - """Fallback when none of the other categories clearly applies.""" - return {"action": "needs_review", "service": service, "summary": summary} - -# --- Redaction: nothing PII-bearing should reach the model or the cache --- -_REDACTIONS = [ - (re.compile(r"[\w.+-]+@[\w-]+\.[\w.-]+"), ""), - (re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b"), ""), - (re.compile(r"\b[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}\b"), ""), - (re.compile(r"\beyJ[\w-]+\.[\w-]+\.[\w-]+\b"), ""), - (re.compile(r"\b[0-9a-fA-F]{16,}\b"), ""), - (re.compile(r"([?&][\w.-]+=)[^&\s|]+"), r"\1"), -] - -def redact(text: str) -> str: - for rx, sub in _REDACTIONS: - text = rx.sub(sub, text) - return text[:1000] - -# Cache keyed on the redacted text: repeated signatures cost nothing. -CACHE_MAX = int(os.environ.get("NEEDLE_CACHE_MAX", 2000)) -_cache = {} -_stats = {"hits": 0, "misses": 0} - -# Initialize Needle Agent once -print(f"Loading Needle 3 model (Cactus-Compute/needle3)...") -t0 = time.perf_counter() -agent = needle.Needle(tools=[ - report_scanner_probe, - report_dependency_unreachable, - report_client_credential_issue, - report_stale_reference, - report_service_defect, - report_capacity_saturation, - report_expected_lifecycle, - report_needs_review, -]) -print(f"Needle 3 loaded in {(time.perf_counter() - t0):.2f}s") - - -class NeedleHandler(BaseHTTPRequestHandler): - def _send_cors(self): - self.send_header("Access-Control-Allow-Origin", "*") - self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") - self.send_header("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Otel-Agent-Token") - - def do_OPTIONS(self): - self.send_response(204) - self._send_cors() - self.end_headers() - - def do_GET(self): - if self.path == "/health" or self.path == "/api/needle/status": - self.send_response(200) - self._send_cors() - self.send_header("Content-Type", "application/json") - self.end_headers() - resp = { - "status": "ok", - "available": True, - "model": "Cactus-Compute/needle3", - "cache": {**_stats, "size": len(_cache)}, - "tools": [ - "scanner_probe", "dependency_unreachable", "client_credential_issue", "stale_reference", - "service_defect", "capacity_saturation", "expected_lifecycle", "needs_review" - ] - } - self.wfile.write(json.dumps(resp).encode("utf-8")) - else: - self.send_response(404) - self._send_cors() - self.end_headers() - - def do_POST(self): - if self.path in ("/classify", "/api/needle/classify"): - content_len = int(self.headers.get("Content-Length", 0)) - body = self.rfile.read(content_len) - try: - data = json.loads(body) if body else {} - except Exception as e: - self.send_response(400) - self._send_cors() - self.send_header("Content-Type", "application/json") - self.end_headers() - self.wfile.write(json.dumps({"error": f"Invalid JSON: {e}"}).encode("utf-8")) - return - - text = data.get("text", "").strip() - # If text is not provided, build from structured fields: - if not text: - parts = [] - if data.get("service"): - parts.append(f"Service: {data['service']}") - if data.get("kind"): - parts.append(f"Kind: {data['kind']}") - if data.get("title"): - parts.append(f"Title: {data['title']}") - if data.get("message"): - parts.append(f"Message: {data['message']}") - if data.get("url"): - parts.append(f"URL: {data['url']}") - if data.get("status_code"): - parts.append(f"Status: {data['status_code']}") - text = " | ".join(parts) - - if not text: - self.send_response(400) - self._send_cors() - self.send_header("Content-Type", "application/json") - self.end_headers() - self.wfile.write(json.dumps({"error": "Missing 'text' or telemetry fields"}).encode("utf-8")) - return - - text = redact(text) - key = hashlib.sha256(text.encode()).hexdigest() - cached = _cache.get(key) - if cached is not None: - _stats["hits"] += 1 - out = {**cached, "cached": True} - self.send_response(200) - self._send_cors() - self.send_header("Content-Type", "application/json") - self.end_headers() - self.wfile.write(json.dumps(out).encode("utf-8")) - return - _stats["misses"] += 1 - - t_start = time.perf_counter() - agent.reset() - res = agent.run(text, strict=False) - latency_ms = (time.perf_counter() - t_start) * 1000 - - results = res.get("results", []) - first_result = results[0] if results and isinstance(results[0], dict) else {} - action = first_result.get("action", "needs_review") - - calls = res.get("function_calls", []) - tool_name = calls[0].get("name") if calls else "triage_tool" - - out = { - "success": True, - "input_text": text, - "action": action, - "tool": tool_name, - "arguments": first_result, - "confidence": float(res.get("confidence", 0.0)), - "reasoning": res.get("reasoning", ""), - "latency_ms": round(latency_ms, 2), - "model": "Cactus-Compute/needle3" - } - - if len(_cache) >= CACHE_MAX: - _cache.pop(next(iter(_cache))) - _cache[key] = out - - self.send_response(200) - self._send_cors() - self.send_header("Content-Type", "application/json") - self.end_headers() - self.wfile.write(json.dumps(out).encode("utf-8")) - else: - self.send_response(404) - self._send_cors() - self.end_headers() - - def log_message(self, format, *args): - # Concise logging - print(f"[needle-sidecar] {args[0]} {args[1]} -> {args[2]}") - - -def run(): - server = ThreadingHTTPServer((HOST, PORT), NeedleHandler) - print(f"[needle-sidecar] Listening on http://{HOST}:{PORT}") - try: - server.serve_forever() - except KeyboardInterrupt: - print("\nShutting down needle sidecar...") - server.server_close() - - -if __name__ == "__main__": - run() diff --git a/cmd/server/main.go b/cmd/server/main.go index 5956ec9..f987020 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -490,8 +490,6 @@ func main() { uiMux.HandleFunc("POST /api/sourcemaps", coll.HandleSourceMap) uiMux.Handle("GET /api/sourcemaps", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleListSourceMaps))) uiMux.Handle("POST /api/sourcemaps/resolve", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleResolveStacks))) - uiMux.HandleFunc("GET /api/needle/status", coll.HandleNeedleStatus) - uiMux.HandleFunc("POST /api/needle/classify", coll.HandleNeedleClassify) // Protected API & WebSocket routes uiMux.Handle("/api/history", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleHistory))) diff --git a/cmd/spike-alerts/main.go b/cmd/spike-alerts/main.go index 742d483..3b8de36 100644 --- a/cmd/spike-alerts/main.go +++ b/cmd/spike-alerts/main.go @@ -1,16 +1,13 @@ // spike-alerts replays the deterministic spike rule over a bounded window of -// normalized_events and, optionally, asks the Needle sidecar what kind of -// failure each top pattern is. It only reads ClickHouse. +// normalized_events. It only reads ClickHouse. package main import ( - "bytes" "context" "encoding/json" "flag" "fmt" "log" - "net/http" "os" "strings" "time" @@ -36,7 +33,6 @@ func main() { minExcess := flag.Int("min-excess", 5, "minimum excess errors to alert") multiple := flag.Float64("multiple", 3, "pattern must be novel or >= multiple * its baseline") cooldown := flag.Duration("cooldown", 30*time.Minute, "suppress repeats of a pattern for this long unless its count doubles") - needle := flag.String("needle", "", "Needle sidecar URL (dormant: it is not accurate enough to gate alerts); empty skips it") showMuted := flag.Bool("show-muted", false, "also report non-actionable categories (probes, stale references, expected lifecycle)") asJSON := flag.Bool("json", false, "emit JSON lines instead of text") flag.Parse() @@ -75,7 +71,6 @@ func main() { log.Printf("bad-actor events (ignored for alerts, kept as data): %v of %d", badActors, len(events)) log.Printf("loaded %d error events", len(events)) - cache := map[string]classification{} first := start.Add(time.Duration(*baselineWindows) * *window) suppress := features.NewSuppressor(*cooldown) muted := map[triage.Category]int{} @@ -87,24 +82,11 @@ func main() { } for _, spike := range suppress.Filter(detected) { fired++ - var cls []classification - for _, p := range spike.Patterns { - c := classification{} - if *needle != "" { - if hit, ok := cache[p.Signature]; ok { - c = hit - } else { - c = classify(*needle, needleText(p.Example)) - cache[p.Signature] = c - } - } - cls = append(cls, c) - } if *asJSON { - _ = json.NewEncoder(os.Stdout).Encode(map[string]any{"spike": spike, "classifications": cls}) + _ = json.NewEncoder(os.Stdout).Encode(map[string]any{"spike": spike}) continue } - printSpike(spike, cls) + printSpike(spike) } } for c, n := range muted { @@ -113,28 +95,7 @@ func main() { log.Printf("%d alert(s) between %s and %s", fired, first.Format(time.RFC3339), end.Format(time.RFC3339)) } -type classification struct { - Action string `json:"action"` - Confidence float64 `json:"confidence"` - Cached bool `json:"cached"` - Err string `json:"error,omitempty"` -} - -func classify(base, text string) classification { - body, _ := json.Marshal(map[string]string{"text": text}) - resp, err := (&http.Client{Timeout: 60 * time.Second}).Post(strings.TrimRight(base, "/")+"/classify", "application/json", bytes.NewReader(body)) - if err != nil { - return classification{Action: "?", Err: err.Error()} - } - defer resp.Body.Close() - var out classification - if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { - return classification{Action: "?", Err: err.Error()} - } - return out -} - -func needleText(e features.Event) string { +func eventText(e features.Event) string { parts := []string{"Service: " + e.Service, "Kind: " + e.Kind} if e.SpanName != "" { parts = append(parts, "Title: "+e.SpanName) @@ -151,17 +112,14 @@ func needleText(e features.Event) string { return strings.Join(parts, " | ") } -func printSpike(s features.Spike, cls []classification) { +func printSpike(s features.Spike) { fmt.Printf("\n%s %s errors=%d (baseline %.1f/window, excess %.0f)\n", s.WindowEnd.Format("01-02 15:04"), s.Service, s.ErrorCount, s.BaselineMean, s.Excess) - for i, p := range s.Patterns { + for _, p := range s.Patterns { tag := "rising" if p.Novel { tag = "NEW" } what := string(alerts.TriageOf(p.Example).Category) - if cls[i].Action != "" { - what += fmt.Sprintf(" [needle:%s %.2f]", cls[i].Action, cls[i].Confidence) - } clients := "" if p.Clients > 0 { clients = fmt.Sprintf(" [%d client%s]", p.Clients, map[bool]string{true: "", false: "s"}[p.Clients == 1]) @@ -169,7 +127,7 @@ func printSpike(s features.Spike, cls []classification) { if p.Example.UAClass != "" && uaclass.IsBot(p.Example.UAClass) { clients += " [" + p.Example.UAClass + "]" } - fmt.Printf(" %-6s x%-4d base %-6.1f %-24s %s%s\n", tag, p.Count, p.BaselineMean, what, needleText(p.Example), clients) + fmt.Printf(" %-6s x%-4d base %-6.1f %-24s %s%s\n", tag, p.Count, p.BaselineMean, what, eventText(p.Example), clients) } } diff --git a/cmd/needle-sidecar/eval/golden.jsonl b/cmd/triage-eval/data/golden.jsonl similarity index 100% rename from cmd/needle-sidecar/eval/golden.jsonl rename to cmd/triage-eval/data/golden.jsonl diff --git a/cmd/needle-sidecar/eval/holdout.jsonl b/cmd/triage-eval/data/holdout.jsonl similarity index 100% rename from cmd/needle-sidecar/eval/holdout.jsonl rename to cmd/triage-eval/data/holdout.jsonl diff --git a/docs/ML.md b/docs/ML.md index d3a5dda..6bad9eb 100644 --- a/docs/ML.md +++ b/docs/ML.md @@ -1,7 +1,7 @@ # Error detection model > Latest status, decisions and next steps: [STATUS.md](STATUS.md). Where they -> differ (ClickHouse is now a LoadBalancer; Needle is dormant), STATUS.md wins. +> differ (ClickHouse is now a LoadBalancer; Needle is removed), STATUS.md wins. ## Current handoff (2026-09-18) diff --git a/docs/STATUS.md b/docs/STATUS.md index 320a057..cf7445c 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -10,7 +10,6 @@ unless stated. Greenfield, single user: radical changes are fine. patched live. `docs/ML.md` still says "keep ClusterIP"; that is superseded. - Credentials are in `.env` (gitignored, mode 600). Load with `set -a; . ./.env; set +a`. `clickhouse-client` is installed locally. -- `.env` deliberately does not enable Needle; pass `-needle URL` explicitly. ## What exists now @@ -21,7 +20,7 @@ unless stated. Greenfield, single user: radical changes are fine. | 404/410 are not errors | `isErrorStatus` in `internal/normalization/normalizer.go` | Done. Rows stored before this still count 404s as errors. | | Backfill | `cmd/normalized-backfill` | Ran 2026-09-19 12:00 to ~21:00 UTC. Queries need `FINAL` (duplicates until merge). | | Spike rule | `internal/features/spike.go` | Excess-error rule, trimmed-mean baseline, `Suppressor` (cooldown, 2x escalation). Tested. | -| Replay CLI | `cmd/spike-alerts` | Replays a range; mutes non-actionable categories; Needle optional and off. | +| Replay CLI | `cmd/spike-alerts` | Replays a range; mutes non-actionable categories. | | Cause categories | `internal/triage` | Deterministic rules, first match wins. Tested. | | Rules scoring | `cmd/triage-eval` | Scores a labelled JSONL; also prepares rows for labelling. | | Offender report | `cmd/bad-actors` | Read-only. Ranks IPs from raw nginx spans, marks `[BLOCK]` / `[INTERNAL]`. | @@ -35,14 +34,14 @@ Categories: `scanner_probe`, `dependency_unreachable`, `client_credential_issue` ## Findings worth remembering -- **Needle scored 10% on the 40-row golden set** (about chance for 8 classes). - Reversing the tool order changed its answers wholesale (15%, 34/40 `service_defect`), - so it picks by position, not content. Confidence is uncorrelated with - correctness. Decision: **Needle stays dormant.** Intended later use: a gate that - decides whether a larger model should act on an issue (affects anyone? fixable by - a smart model?). Do not use it for categorization. +- **Needle was removed (2026-11-01).** It scored 10% on the 40-row golden set (about + chance for 8 classes), picked by tool position rather than content, and its + confidence was uncorrelated with correctness. Sidecar, proxy handlers, UI and k8s + container are gone; the labelled sets live in `cmd/triage-eval/data/`. +- **Edge 4xx never pages.** nginx 4xx (registry 401 challenges, `/oauth2/auth`, + scanner 405s) is dropped in `FilterActionable`; nginx 5xx still alerts. - **Rules scored 39/40 on the golden set, but that is a fit** (written after seeing - the rows). A fresh 60-row holdout is in `cmd/needle-sidecar/eval/holdout.jsonl` + the rows). A fresh 60-row holdout is in `cmd/triage-eval/data/holdout.jsonl` (from 2026-09-18, unseen); it still needs your `expected` labels. Known probe-like paths that still land in `stale_reference`: `/ssl/server.key`, `/_debugbar/open`, `/debug/pprof`, `/actuator/*`, `/graphql`, `/server-info`, @@ -78,7 +77,7 @@ Categories: `scanner_probe`, `dependency_unreachable`, `client_credential_issue` 1. Deploy the collector so normalized rows stay current; verify with `SELECT max(timestamp) FROM otel.normalized_events`. 2. Label `holdout.jsonl` (fill `expected`), run - `go run ./cmd/triage-eval -input cmd/needle-sidecar/eval/holdout.jsonl`, fix + `go run ./cmd/triage-eval -input cmd/triage-eval/data/holdout.jsonl`, fix the rule families that miss. Then re-export a fresh holdout so scoring stays honest. 3. Warm-up guard in `DetectSpikes`: require baseline data before alerting (the @@ -164,7 +163,7 @@ Categories: `scanner_probe`, `dependency_unreachable`, `client_credential_issue` ```bash set -a; . ./.env; set +a -# replay the spike rule over a range (Needle off) +# replay the spike rule over a range go run ./cmd/spike-alerts -since 2026-09-19T12:00:00Z -until 2026-09-19T20:30:00Z -baseline-windows 12 # add -show-muted to see probes / stale / lifecycle patterns too @@ -172,12 +171,7 @@ go run ./cmd/spike-alerts -since 2026-09-19T12:00:00Z -until 2026-09-19T20:30:00 go run ./cmd/bad-actors -since 2026-09-19T08:00:00Z -until 2026-09-19T20:30:00Z # score the rules against labelled rows -go run ./cmd/triage-eval -input cmd/needle-sidecar/eval/holdout.jsonl - -# Needle eval (needs the sidecar running; venv was in scratch and is gone) -pip install cactus-needle # python 3.12 recommended -NEEDLE_PORT=5005 NEEDLE_HOST=127.0.0.1 python cmd/needle-sidecar/server.py -python cmd/needle-sidecar/eval/run_eval.py +go run ./cmd/triage-eval -input cmd/triage-eval/data/holdout.jsonl ``` Backfill in small windows only (10 to 30 minutes over the LAN); one-hour windows @@ -187,14 +181,11 @@ through a `kubectl port-forward` hung and dropped. Everything is uncommitted. Files changed by this work: `internal/{storage, normalization,features,triage,honeypot,blocker}`, `cmd/{spike-alerts, -triage-eval,bad-actors,normalized-backfill,server,needle-sidecar}`, +triage-eval,bad-actors,normalized-backfill,server}`, `k8s/{clickhouse,deployment}.yaml`, `.gitignore`. **Not from this work; check before committing:** `AGENTS.md`, -`internal/collector/{collector_test,issue_extract}.go`, `otel-ui/src/**` -(including a deleted `GaugeMeter.tsx`). Also tracked `__pycache__/*.pyc` files -under `cmd/needle-sidecar` show as modified; they should be untracked and -gitignored. +`internal/collector/{collector_test,issue_extract}.go`, `otel-ui/src/**` (including a deleted `GaugeMeter.tsx`). ## IP details (MaxMind) diff --git a/index.html b/index.html index 2ec9bdb..00e424e 100644 --- a/index.html +++ b/index.html @@ -141,45 +141,9 @@ Clear Logs - -
-

Needle 3 AI Telemetry Triage

-

Test on-device telemetry classification with Cactus-Compute/needle3:

-
- - -
-
- Click "Classify with Needle 3" to test the sidecar... -
-